mutineer 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +466 -0
- data/README.md +157 -747
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/cli.rb +132 -28
- data/lib/mutineer/concern_blocks.rb +187 -5
- data/lib/mutineer/config.rb +6 -0
- data/lib/mutineer/coverage_map.rb +92 -17
- data/lib/mutineer/daemon_backend.rb +75 -32
- data/lib/mutineer/daemon_client.rb +50 -17
- data/lib/mutineer/daemon_server.rb +93 -41
- data/lib/mutineer/external_backend.rb +150 -17
- data/lib/mutineer/init.rb +1 -1
- data/lib/mutineer/isolation.rb +70 -25
- data/lib/mutineer/job_plan.rb +119 -34
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +10 -8
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +22 -0
- data/lib/mutineer/mutators/call_to_receiver.rb +20 -7
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +18 -6
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +226 -12
- data/lib/mutineer/rails_worker_db.rb +200 -16
- data/lib/mutineer/redaction.rb +112 -11
- data/lib/mutineer/reporter.rb +109 -21
- data/lib/mutineer/result.rb +33 -5
- data/lib/mutineer/runner.rb +126 -18
- data/lib/mutineer/statement_lines.rb +14 -8
- data/lib/mutineer/subject.rb +13 -5
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners.rb +4 -2
- data/lib/mutineer/triage.rb +2 -2
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/skills/mutineer/SKILL.md +84 -0
- metadata +5 -1
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "outer_reporter"
|
|
4
|
+
require_relative "../redaction"
|
|
5
|
+
|
|
6
|
+
module Mutineer
|
|
7
|
+
class MinitestIntegration
|
|
8
|
+
# Scores a run as `error`, not `killed`, when its first failing test failed
|
|
9
|
+
# because of an Active Record connection or lock error (#296). Such a
|
|
10
|
+
# failure says the database broke, not that a test caught the mutant.
|
|
11
|
+
# Child-process only, like MinitestIntegration.
|
|
12
|
+
#
|
|
13
|
+
# Only the first failed result on the outer reporter counts (see
|
|
14
|
+
# {OuterReporter}). A run that stops at its first failure ends there, so a
|
|
15
|
+
# full run and a `--matrix` run give the same verdict. One limit: the tests
|
|
16
|
+
# of a `parallelize_me!` class record in the order their threads finish, so
|
|
17
|
+
# a mutant that makes one parallel test hit a database error and another
|
|
18
|
+
# fail an assertion can score `error` in one run and `killed` in the next.
|
|
19
|
+
# Stdlib only: the app-side daemon loads this file without Prism. An
|
|
20
|
+
# unknown Minitest shape arms nothing, and the run keeps its 0 or 1.
|
|
21
|
+
module InTestDatabaseError
|
|
22
|
+
extend OuterReporter::Client
|
|
23
|
+
|
|
24
|
+
# The Active Record errors that mean the database, not the test, failed.
|
|
25
|
+
# Subclasses count too: `ConnectionTimeoutError` (a pool timeout) and
|
|
26
|
+
# `DatabaseConnectionError` are `ConnectionNotEstablished`. Looked up by
|
|
27
|
+
# name when a result is recorded, so a project without Active Record, or
|
|
28
|
+
# a Rails without `ConnectionFailed` (before 7.1), matches nothing. Other
|
|
29
|
+
# `StatementInvalid` errors are not here: a mutant can cause bad SQL.
|
|
30
|
+
NAMES = %w[
|
|
31
|
+
ActiveRecord::ConnectionNotEstablished
|
|
32
|
+
ActiveRecord::ConnectionFailed
|
|
33
|
+
ActiveRecord::Deadlocked
|
|
34
|
+
ActiveRecord::LockWaitTimeout
|
|
35
|
+
].freeze
|
|
36
|
+
|
|
37
|
+
# Subclasses of {NAMES} that point at the code, not the database. Rails 8
|
|
38
|
+
# raises `ConnectionNotDefined` for a role or shard that no connection
|
|
39
|
+
# defines, which a mutant of a `connected_to` call can cause. Rails 7.2
|
|
40
|
+
# raises a plain `ConnectionNotEstablished` there, so on 7.2 such a mutant
|
|
41
|
+
# scores `error`. The message is not matched (plan Q2).
|
|
42
|
+
CODE_ERROR_NAMES = %w[ActiveRecord::ConnectionNotDefined].freeze
|
|
43
|
+
|
|
44
|
+
# The lock errors in {NAMES}, which get their own stderr hint.
|
|
45
|
+
LOCK_ERROR_NAMES = %w[ActiveRecord::Deadlocked ActiveRecord::LockWaitTimeout].freeze
|
|
46
|
+
|
|
47
|
+
# Prepended on `Minitest::CompositeReporter`.
|
|
48
|
+
module RecordFirstFailure
|
|
49
|
+
# Records the result, then lets the module keep the first failure.
|
|
50
|
+
#
|
|
51
|
+
# @param result [Minitest::Result] the result of one test.
|
|
52
|
+
# @return [void]
|
|
53
|
+
def record(result)
|
|
54
|
+
super
|
|
55
|
+
InTestDatabaseError.record(self, result)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
class << self
|
|
60
|
+
# The first failed result's database error, false when that result
|
|
61
|
+
# failed for another reason, or nil before any failure.
|
|
62
|
+
#
|
|
63
|
+
# @return [Exception, false, nil]
|
|
64
|
+
attr_accessor :first_failure
|
|
65
|
+
|
|
66
|
+
# Installs the hooks and arms the module for this process.
|
|
67
|
+
#
|
|
68
|
+
# @return [Boolean] false when the Minitest shape is unknown.
|
|
69
|
+
def arm!
|
|
70
|
+
outer = OuterReporter.hook_for_loaded_minitest
|
|
71
|
+
return false unless outer
|
|
72
|
+
|
|
73
|
+
OuterReporter.prepend_once(::Minitest::CompositeReporter, RecordFirstFailure)
|
|
74
|
+
OuterReporter.prepend_once(::Minitest.singleton_class, outer)
|
|
75
|
+
OuterReporter.register(self)
|
|
76
|
+
self.armed_pid = Process.pid
|
|
77
|
+
self.armed_reporter = nil
|
|
78
|
+
self.first_failure = nil
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Clears the armed state.
|
|
83
|
+
#
|
|
84
|
+
# @return [void]
|
|
85
|
+
def disarm!
|
|
86
|
+
self.armed_pid = nil
|
|
87
|
+
self.armed_reporter = nil
|
|
88
|
+
self.first_failure = nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Keeps the first failed result recorded on the outer reporter. Minitest
|
|
92
|
+
# records from several threads only under its own reporter lock. A
|
|
93
|
+
# result it cannot read counts as an ordinary failure, so a fault here
|
|
94
|
+
# keeps the old verdict and never turns a kill into an error.
|
|
95
|
+
#
|
|
96
|
+
# @param reporter [Minitest::CompositeReporter] the recording reporter.
|
|
97
|
+
# @param result [Minitest::Result] the result of one test.
|
|
98
|
+
# @return [void]
|
|
99
|
+
def record(reporter, result)
|
|
100
|
+
return if result.passed? || result.skipped?
|
|
101
|
+
return unless first_failure.nil? && outer_reporter?(reporter)
|
|
102
|
+
|
|
103
|
+
self.first_failure = database_error(result.failures.first) || false
|
|
104
|
+
rescue StandardError
|
|
105
|
+
self.first_failure = false
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The run's exit code: 2 instead of 1 when the armed run's first failure
|
|
109
|
+
# was a database error. Then it also writes one line on stderr that
|
|
110
|
+
# names the error. `STDERR`, not `warn`: a test can leave `$stderr` as
|
|
111
|
+
# a `StringIO`. The flush matters because both fork boundaries end with
|
|
112
|
+
# `exit!`.
|
|
113
|
+
#
|
|
114
|
+
# @param code [Integer] 0 or 1 from the run.
|
|
115
|
+
# @return [Integer] `code`, or 2.
|
|
116
|
+
def exit_code(code)
|
|
117
|
+
error = first_failure if armed_here?
|
|
118
|
+
return code unless code == 1 && error
|
|
119
|
+
|
|
120
|
+
STDERR.puts(line(error))
|
|
121
|
+
STDERR.flush
|
|
122
|
+
2
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# The database error a failure wraps. The first entry of a result is
|
|
126
|
+
# the error that ended the test: Minitest records the setup or body
|
|
127
|
+
# error before a teardown error. The cause chain is not read: Active
|
|
128
|
+
# Record already translates driver errors into the listed classes.
|
|
129
|
+
#
|
|
130
|
+
# Minitest before 5.14 names the wrapped error `exception`.
|
|
131
|
+
#
|
|
132
|
+
# Shared with the matrix recorder so an unavailable database outcome
|
|
133
|
+
# cannot be named as a mutation kill.
|
|
134
|
+
#
|
|
135
|
+
# @api private
|
|
136
|
+
# @param failure [Minitest::Assertion, nil] the result's first entry.
|
|
137
|
+
# @return [Exception, nil]
|
|
138
|
+
def database_error(failure)
|
|
139
|
+
return unless failure.is_a?(::Minitest::UnexpectedError)
|
|
140
|
+
|
|
141
|
+
error = failure.respond_to?(:error) ? failure.error : failure.exception
|
|
142
|
+
return if loaded(CODE_ERROR_NAMES).any? { |klass| error.is_a?(klass) }
|
|
143
|
+
|
|
144
|
+
error if loaded(NAMES).any? { |klass| error.is_a?(klass) }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
private
|
|
148
|
+
|
|
149
|
+
# The named classes this process has loaded.
|
|
150
|
+
#
|
|
151
|
+
# @param names [Array<String>] constant names.
|
|
152
|
+
# @return [Array<Class>]
|
|
153
|
+
def loaded(names)
|
|
154
|
+
names.filter_map { |name| Object.const_get(name) if Object.const_defined?(name) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The stderr line. Redacted before the cut to one line, so a cut can
|
|
158
|
+
# never split a password away from its URL. The wording fits every run,
|
|
159
|
+
# the clean check included, and a mutant can cause the error too.
|
|
160
|
+
#
|
|
161
|
+
# @param error [Exception] the database error.
|
|
162
|
+
# @return [String]
|
|
163
|
+
def line(error)
|
|
164
|
+
text = Redaction.text("#{error.class}: #{error.message}".scrub)
|
|
165
|
+
text = text.lines.first.to_s.chomp.delete_suffix(".")
|
|
166
|
+
"[mutineer-child] a test failed with a database error, so the run has no verdict: #{text}. #{hint(error)}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# What to check for the error: a lock error comes from a second
|
|
170
|
+
# connection or thread, any other from the server.
|
|
171
|
+
#
|
|
172
|
+
# @param error [Exception] the database error.
|
|
173
|
+
# @return [String]
|
|
174
|
+
def hint(error)
|
|
175
|
+
if loaded(LOCK_ERROR_NAMES).any? { |klass| error.is_a?(klass) }
|
|
176
|
+
"Look for a test or app code that holds a lock on a second connection or thread."
|
|
177
|
+
else
|
|
178
|
+
"Check that the database server is up and has free connections."
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "outer_reporter"
|
|
4
|
+
require_relative "in_test_database_error"
|
|
4
5
|
require_relative "../kill_channel"
|
|
5
6
|
|
|
6
7
|
module Mutineer
|
|
@@ -22,7 +23,7 @@ module Mutineer
|
|
|
22
23
|
# Prepended on each loaded test class: one test's run. Serial and parallel
|
|
23
24
|
# runs both call it, in Minitest 5 and 6.
|
|
24
25
|
module MarkSkippedTest
|
|
25
|
-
# Runs the test inside a `skip` region once a serial test has
|
|
26
|
+
# Runs the test inside a `skip` region once a serial test has failed.
|
|
26
27
|
#
|
|
27
28
|
# @return [Minitest::Result]
|
|
28
29
|
def run
|
|
@@ -35,7 +36,7 @@ module Mutineer
|
|
|
35
36
|
# A run without `--matrix` skips both after a stop (see
|
|
36
37
|
# StopAtFirstFailure::SkipAfterStop6).
|
|
37
38
|
module MarkSkippedClass6
|
|
38
|
-
# Runs the class inside a `skip` region once a serial test has
|
|
39
|
+
# Runs the class inside a `skip` region once a serial test has failed.
|
|
39
40
|
#
|
|
40
41
|
# @param reporter [Minitest::CompositeReporter]
|
|
41
42
|
# @param options [Hash]
|
|
@@ -46,7 +47,7 @@ module Mutineer
|
|
|
46
47
|
end
|
|
47
48
|
|
|
48
49
|
# Runs one test's class-level step inside a `skip` region once a
|
|
49
|
-
# serial test has
|
|
50
|
+
# serial test has failed.
|
|
50
51
|
#
|
|
51
52
|
# @param klass [Class]
|
|
52
53
|
# @param method_name [String]
|
|
@@ -59,7 +60,7 @@ module Mutineer
|
|
|
59
60
|
|
|
60
61
|
# The Minitest 5 form of {MarkSkippedClass6}.
|
|
61
62
|
module MarkSkippedClass5
|
|
62
|
-
# Runs the class inside a `skip` region once a serial test has
|
|
63
|
+
# Runs the class inside a `skip` region once a serial test has failed.
|
|
63
64
|
#
|
|
64
65
|
# @param reporter [Minitest::CompositeReporter]
|
|
65
66
|
# @param options [Hash]
|
|
@@ -70,7 +71,7 @@ module Mutineer
|
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
# Runs one test's class-level step inside a `skip` region once a
|
|
73
|
-
# serial test has
|
|
74
|
+
# serial test has failed.
|
|
74
75
|
#
|
|
75
76
|
# @param klass [Class]
|
|
76
77
|
# @param method_name [String]
|
|
@@ -114,11 +115,11 @@ module Mutineer
|
|
|
114
115
|
# @return [Integer, nil]
|
|
115
116
|
attr_accessor :seen
|
|
116
117
|
|
|
117
|
-
# True once a serial test
|
|
118
|
-
# skips each later test and class.
|
|
118
|
+
# True once a serial test failed, including a database failure: from
|
|
119
|
+
# then on, a run without `--matrix` skips each later test and class.
|
|
119
120
|
#
|
|
120
121
|
# @return [Boolean, nil]
|
|
121
|
-
attr_accessor :
|
|
122
|
+
attr_accessor :serial_failed
|
|
122
123
|
|
|
123
124
|
# Installs the hooks, arms the recorder for this process, and writes the
|
|
124
125
|
# channel's `start` line. Call it after the test files load, so the
|
|
@@ -149,7 +150,7 @@ module Mutineer
|
|
|
149
150
|
# the run joins it without running, which would break the count.
|
|
150
151
|
self.runnables = runnables.dup
|
|
151
152
|
self.seen = 0
|
|
152
|
-
self.
|
|
153
|
+
self.serial_failed = false
|
|
153
154
|
KillChannel.write_start(channel)
|
|
154
155
|
true
|
|
155
156
|
end
|
|
@@ -186,7 +187,7 @@ module Mutineer
|
|
|
186
187
|
self.armed_reporter = nil
|
|
187
188
|
self.channel = nil
|
|
188
189
|
self.runnables = nil
|
|
189
|
-
self.
|
|
190
|
+
self.serial_failed = false
|
|
190
191
|
end
|
|
191
192
|
|
|
192
193
|
# Writes the `parallel` line when the first parallel class starts on the
|
|
@@ -206,7 +207,7 @@ module Mutineer
|
|
|
206
207
|
end
|
|
207
208
|
|
|
208
209
|
# Runs the block between `skip` and `unskip` when a serial test has
|
|
209
|
-
#
|
|
210
|
+
# failed in this armed process, else just runs it. No `ensure`: a block
|
|
210
211
|
# that never returns (an exit, a crash) must leave its region open, so
|
|
211
212
|
# the parent knows the child ended where a run without `--matrix`
|
|
212
213
|
# would not have been.
|
|
@@ -214,7 +215,7 @@ module Mutineer
|
|
|
214
215
|
# @yieldreturn [Object]
|
|
215
216
|
# @return [Object] the block's value.
|
|
216
217
|
def skipping
|
|
217
|
-
open = channel &&
|
|
218
|
+
open = channel && serial_failed && armed_here?
|
|
218
219
|
KillChannel.write_skip(channel) if open
|
|
219
220
|
value = yield
|
|
220
221
|
KillChannel.write_unskip(channel) if open
|
|
@@ -222,9 +223,10 @@ module Mutineer
|
|
|
222
223
|
end
|
|
223
224
|
|
|
224
225
|
# Sends one result recorded on `reporter`: a pass, or a kill for a
|
|
225
|
-
# failure or an error
|
|
226
|
-
# seen, and results on other
|
|
227
|
-
# `parallel` line goes out when the
|
|
226
|
+
# failure or an error, or an unavailable database outcome. Skips send
|
|
227
|
+
# no test line, though they count as seen, and results on other
|
|
228
|
+
# reporters count for nothing. The `parallel` line goes out when the
|
|
229
|
+
# first parallel class starts (see
|
|
228
230
|
# {.class_starting}), so a kill that precedes it came from a serial test.
|
|
229
231
|
#
|
|
230
232
|
# @param reporter [Minitest::CompositeReporter] the recording reporter.
|
|
@@ -251,8 +253,14 @@ module Mutineer
|
|
|
251
253
|
def send_result(result)
|
|
252
254
|
return if result.skipped?
|
|
253
255
|
|
|
254
|
-
event = result.passed?
|
|
255
|
-
|
|
256
|
+
event = if result.passed?
|
|
257
|
+
KillChannel::PASS
|
|
258
|
+
elsif InTestDatabaseError.database_error(result.failures.first)
|
|
259
|
+
KillChannel::UNAVAILABLE
|
|
260
|
+
else
|
|
261
|
+
KillChannel::KILL
|
|
262
|
+
end
|
|
263
|
+
self.serial_failed = true if event != KillChannel::PASS && !parallel_marked
|
|
256
264
|
file, line = Array(result.source_location)
|
|
257
265
|
KillChannel.write(channel, event, file, *name_and_id(result, line))
|
|
258
266
|
rescue StandardError
|
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "minitest_integration/stop_at_first_failure"
|
|
4
4
|
require_relative "minitest_integration/kill_recorder"
|
|
5
|
+
require_relative "minitest_integration/in_test_database_error"
|
|
5
6
|
|
|
6
7
|
module Mutineer
|
|
7
8
|
# Child-process-only: loads a test file in the current process and runs it
|
|
8
9
|
# programmatically, returning an exit status integer (0 = all passed,
|
|
9
|
-
# 1 = any failure/error
|
|
10
|
+
# 1 = any failure/error, 2 = the first failure was an Active Record
|
|
11
|
+
# connection or lock error).
|
|
10
12
|
#
|
|
11
13
|
# Never call this in the parent — it manipulates global Minitest state
|
|
12
14
|
# (autorun, runnables) that only makes sense in a throwaway forked child.
|
|
@@ -14,15 +16,39 @@ module Mutineer
|
|
|
14
16
|
# A missing minitest is rescued as FrameworkUnavailable (Isolation.run
|
|
15
17
|
# still turns that raise into exit 2). There is no rescue around the
|
|
16
18
|
# suite run itself: Isolation.run's fork block is the single exception
|
|
17
|
-
# boundary for unexpected errors. Swallowing those here would
|
|
18
|
-
#
|
|
19
|
+
# boundary for unexpected errors. Swallowing those here would hide a
|
|
20
|
+
# crash as a verdict. The one 2 this method returns on purpose comes from
|
|
21
|
+
# Minitest's own results: the run's first failing test failed with an
|
|
22
|
+
# Active Record connection or lock error (#296, {InTestDatabaseError}).
|
|
19
23
|
class MinitestIntegration
|
|
20
|
-
# The seed of
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
+
# The seed of every Minitest run Mutineer starts, unless `SEED` is set:
|
|
25
|
+
# coverage capture, the clean checks and the mutant runs (#282). A file's
|
|
26
|
+
# clean check then runs its tests in the order a mutant run of that file
|
|
27
|
+
# uses. Several covering files can still differ from the combined clean
|
|
28
|
+
# check: a smaller class list shuffles differently. With the stop,
|
|
29
|
+
# the test order can decide `killed` vs `timeout`; a fixed order keeps the
|
|
30
|
+
# verdict stable for a `--baseline` gate. A `--matrix` run uses the same
|
|
24
31
|
# order, so its first failure comes at the same point as the stop's.
|
|
25
|
-
|
|
32
|
+
#
|
|
33
|
+
# @api private
|
|
34
|
+
PINNED_SEED = 1
|
|
35
|
+
|
|
36
|
+
# The arguments that give `Minitest.run` the shared seed. With `SEED` set,
|
|
37
|
+
# none: Minitest reads `SEED` itself.
|
|
38
|
+
#
|
|
39
|
+
# @api private
|
|
40
|
+
# @return [Array<String>] `["--seed", "1"]`, or `[]` when `SEED` is set.
|
|
41
|
+
def self.seed_args
|
|
42
|
+
ENV["SEED"] ? [] : ["--seed", seed]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The seed the Minitest runs use: `SEED` when set, else {PINNED_SEED}.
|
|
46
|
+
#
|
|
47
|
+
# @api private
|
|
48
|
+
# @return [String]
|
|
49
|
+
def self.seed
|
|
50
|
+
ENV.fetch("SEED", PINNED_SEED.to_s)
|
|
51
|
+
end
|
|
26
52
|
|
|
27
53
|
# Tested via runner_test.rb, not in isolation — a direct unit test
|
|
28
54
|
# would require forking and duplicate isolation_test's coverage.
|
|
@@ -36,7 +62,8 @@ module Mutineer
|
|
|
36
62
|
# test. Only the mutant path passes true.
|
|
37
63
|
# @param record_to [IO, nil] a `--matrix` run: send each test's outcome to
|
|
38
64
|
# this channel (see {KillRecorder}) and run every test.
|
|
39
|
-
# @return [Integer] 0 on success, 1 on failure
|
|
65
|
+
# @return [Integer] 0 on success, 1 on failure, 2 when the first failure
|
|
66
|
+
# was an Active Record connection or lock error (see {InTestDatabaseError}).
|
|
40
67
|
# @raise [ArgumentError] when both options are given: a matrix run never stops.
|
|
41
68
|
def self.run(test_files, stop_at_first_failure: false, record_to: nil)
|
|
42
69
|
if stop_at_first_failure && record_to
|
|
@@ -75,22 +102,25 @@ module Mutineer
|
|
|
75
102
|
elsif stop_at_first_failure
|
|
76
103
|
StopAtFirstFailure.arm!(Minitest::Runnable.runnables)
|
|
77
104
|
end
|
|
78
|
-
#
|
|
79
|
-
#
|
|
80
|
-
args = armed && !ENV["SEED"] ? ["--seed", STOP_AT_FIRST_FAILURE_SEED.to_s] : []
|
|
105
|
+
# An unknown Minitest shape arms no hook and gets a full run, in the
|
|
106
|
+
# same seeded order.
|
|
81
107
|
keep_file_order(rank) if armed
|
|
108
|
+
# Every run, on its own line: `armed` decides the file order (#296).
|
|
109
|
+
InTestDatabaseError.arm!
|
|
82
110
|
# No silencing here: the fork boundary that calls this method has already
|
|
83
111
|
# pointed stdout at File::NULL (see ChildStdout).
|
|
84
|
-
passed = Minitest.run(
|
|
112
|
+
passed = Minitest.run(seed_args)
|
|
85
113
|
# Reached only when the suite returned: a test that exits the process
|
|
86
114
|
# skips it, and the parent then keeps the row incomplete.
|
|
87
115
|
KillRecorder.finish! if record_to
|
|
88
116
|
|
|
89
117
|
# A plugin can replace the summary reporter, so a stop decides by itself.
|
|
90
|
-
passed && !StopAtFirstFailure.stopped_here? ? 0 : 1
|
|
118
|
+
code = passed && !StopAtFirstFailure.stopped_here? ? 0 : 1
|
|
119
|
+
InTestDatabaseError.exit_code(code)
|
|
91
120
|
ensure
|
|
92
121
|
StopAtFirstFailure.disarm!
|
|
93
122
|
KillRecorder.disarm!
|
|
123
|
+
InTestDatabaseError.disarm!
|
|
94
124
|
end
|
|
95
125
|
|
|
96
126
|
# Runs the test classes in the order of their files (#203). Minitest
|
|
@@ -25,12 +25,14 @@ module Mutineer
|
|
|
25
25
|
#
|
|
26
26
|
# The operator skips an implicit array (`x = 1, 2`), because it has no
|
|
27
27
|
# brackets. It also skips an array that holds a heredoc, because the
|
|
28
|
-
# heredoc body stays behind as code.
|
|
28
|
+
# heredoc body stays behind as code. Only the heredoc skip counts as
|
|
29
|
+
# declined.
|
|
29
30
|
#
|
|
30
31
|
# @param node [Prism::ArrayNode] array node to inspect.
|
|
31
32
|
# @return [void]
|
|
32
33
|
def emit(node)
|
|
33
|
-
return if node.opening_loc.nil? || node.elements.empty?
|
|
34
|
+
return if node.opening_loc.nil? || node.elements.empty?
|
|
35
|
+
return decline if heredoc?(node)
|
|
34
36
|
|
|
35
37
|
@mutations << Mutation.new(
|
|
36
38
|
start_offset: node.location.start_offset,
|
|
@@ -20,6 +20,7 @@ module Mutineer
|
|
|
20
20
|
def mutations_for(subject, source)
|
|
21
21
|
@source = source
|
|
22
22
|
@mutations = []
|
|
23
|
+
@declined = 0
|
|
23
24
|
@body = subject.def_node.body
|
|
24
25
|
@body&.accept(self)
|
|
25
26
|
drop_dangling_heredocs
|
|
@@ -50,8 +51,26 @@ module Mutineer
|
|
|
50
51
|
@in_undiscovered = outer
|
|
51
52
|
end
|
|
52
53
|
|
|
54
|
+
# Returns how many candidates the last {#mutations_for} call declined: an
|
|
55
|
+
# edit that would split a heredoc from its body, or an equivalent skip
|
|
56
|
+
# listed in docs/equivalent-skips.md. A declined candidate is never a
|
|
57
|
+
# mutation, so it is not in the run's total.
|
|
58
|
+
#
|
|
59
|
+
# @return [Integer] declined candidates; 0 before the first call.
|
|
60
|
+
def declined
|
|
61
|
+
@declined || 0
|
|
62
|
+
end
|
|
63
|
+
|
|
53
64
|
private
|
|
54
65
|
|
|
66
|
+
# Counts one declined candidate.
|
|
67
|
+
#
|
|
68
|
+
# @return [nil] so a skip line reads `return decline if ...`.
|
|
69
|
+
def decline
|
|
70
|
+
@declined += 1
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
55
74
|
# Drops a mutation that splits a heredoc opener from its body.
|
|
56
75
|
#
|
|
57
76
|
# The body sits past the opener's node. Replacing the opener with text
|
|
@@ -61,10 +80,13 @@ module Mutineer
|
|
|
61
80
|
# the replaced range holds the body but the replacement keeps only the
|
|
62
81
|
# opener, the opener has no body and the source does not parse. A
|
|
63
82
|
# replacement that copies the opener and the body together stays.
|
|
83
|
+
# Each dropped mutation counts as declined.
|
|
64
84
|
#
|
|
65
85
|
# @return [void]
|
|
66
86
|
def drop_dangling_heredocs
|
|
87
|
+
before = @mutations.size
|
|
67
88
|
@mutations.reject! { |mutation| dangling_heredoc?(mutation) }
|
|
89
|
+
@declined += before - @mutations.size
|
|
68
90
|
end
|
|
69
91
|
|
|
70
92
|
# Returns whether the mutation replaces a heredoc opener and keeps the
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "base"
|
|
4
|
-
require_relative "chain_link"
|
|
5
4
|
|
|
6
5
|
module Mutineer
|
|
7
6
|
module Mutators
|
|
@@ -21,11 +20,14 @@ module Mutineer
|
|
|
21
20
|
# that is the receiver of a dotted call, or of a call write or a call
|
|
22
21
|
# target, is a link, so it gets no mutant here. The two operators give the
|
|
23
22
|
# same source only when the last call repeats the link before it, as in
|
|
24
|
-
# `node.name.name`.
|
|
23
|
+
# `node.name.name`. When both edits start on the same line, the job plan
|
|
24
|
+
# runs one copy, the `chain_link` one (#275).
|
|
25
25
|
#
|
|
26
|
-
# Calls in {
|
|
27
|
-
#
|
|
28
|
-
#
|
|
26
|
+
# Calls in {SKIPPED} are never replaced; each one counts as declined. The
|
|
27
|
+
# list is shorter than {ChainLink::SKIPPED}: on the last call, a conversion
|
|
28
|
+
# such as `to_s` or `to_i` decides the type of the value that the
|
|
29
|
+
# expression returns, so `m[1].to_i` becomes `m[1]`. `Foo.new` becomes the
|
|
30
|
+
# class, and almost any use of the class raises.
|
|
29
31
|
#
|
|
30
32
|
# A call is not replaced when its name is an operator (`a.+(1)`), or a
|
|
31
33
|
# setter (`a.b = 1`). A call with no dot (`a + b`, `a[0]`) is not replaced.
|
|
@@ -33,6 +35,17 @@ module Mutineer
|
|
|
33
35
|
# Matches the name of a call written as a word, not as an operator.
|
|
34
36
|
WORD_NAME = /\A[[:alpha:]_]/
|
|
35
37
|
|
|
38
|
+
# Method names whose last call is never replaced. A name is here when
|
|
39
|
+
# its mutant is most often equivalent, or when almost any test that runs
|
|
40
|
+
# the line kills it. `freeze` and `itself` return the receiver. `dup`
|
|
41
|
+
# and `clone` return a copy, so the mutant shows when code later changes
|
|
42
|
+
# the copy or the original, or compares the result with the original by
|
|
43
|
+
# identity. Dropping `to_set` most often leaves an Array, which answers
|
|
44
|
+
# `include?` and `empty?` as a Set does (it keeps duplicates, so `size`
|
|
45
|
+
# and `each` can differ). `new` leaves the class in place of an
|
|
46
|
+
# instance, which almost any test that runs the line kills.
|
|
47
|
+
SKIPPED = %i[new dup clone freeze itself to_set].freeze
|
|
48
|
+
|
|
36
49
|
# Resets the per-subject record of calls that are links.
|
|
37
50
|
#
|
|
38
51
|
# @param subject [Mutineer::Subject] subject whose body is visited.
|
|
@@ -104,7 +117,7 @@ module Mutineer
|
|
|
104
117
|
end
|
|
105
118
|
|
|
106
119
|
# Emits the replacement of a call by its receiver, unless the name is an
|
|
107
|
-
# operator, a setter, or in {
|
|
120
|
+
# operator, a setter, or in {SKIPPED}.
|
|
108
121
|
#
|
|
109
122
|
# @param node [Prism::CallNode] the dotted call to replace.
|
|
110
123
|
# @return [void]
|
|
@@ -112,7 +125,7 @@ module Mutineer
|
|
|
112
125
|
name = node.name.to_s
|
|
113
126
|
return unless name.match?(WORD_NAME)
|
|
114
127
|
return if name.end_with?("=")
|
|
115
|
-
return if
|
|
128
|
+
return decline if SKIPPED.include?(node.name)
|
|
116
129
|
|
|
117
130
|
receiver = node.receiver.location
|
|
118
131
|
@mutations << Mutation.new(
|
|
@@ -16,16 +16,16 @@ module Mutineer
|
|
|
16
16
|
# The mutant survives when no test tells the chain apart from the same chain
|
|
17
17
|
# without that step: a scope, a filter or a lookup the tests never see.
|
|
18
18
|
#
|
|
19
|
-
# Links in {SKIPPED} are never dropped
|
|
20
|
-
# target type or needs no copy, a conversion
|
|
21
|
-
# (`list.dup.sort`) is a no-op, so dropping
|
|
22
|
-
# equivalent mutant. Dropping `new` sends the next call to the class, which
|
|
19
|
+
# Links in {SKIPPED} are never dropped; each one counts as declined. On a
|
|
20
|
+
# value that already has the target type or needs no copy, a conversion
|
|
21
|
+
# (`name.to_s.strip`) or a copy (`list.dup.sort`) is a no-op, so dropping
|
|
22
|
+
# it most often makes an equivalent mutant. Dropping `new` sends the next call to the class, which
|
|
23
23
|
# raises (killed by any test that runs the line) or reaches a class method
|
|
24
24
|
# that builds the instance itself (equivalent); neither says anything about
|
|
25
25
|
# the tests.
|
|
26
26
|
class ChainLink < Base
|
|
27
27
|
# Method names whose link is never dropped: core conversions and copies,
|
|
28
|
-
# plus `new`. {CallToReceiver}
|
|
28
|
+
# plus `new`. {CallToReceiver} has its own shorter list for the last call.
|
|
29
29
|
SKIPPED = %i[
|
|
30
30
|
to_s to_str to_sym to_i to_int to_f to_r to_c to_a to_ary to_h to_hash to_proc to_set
|
|
31
31
|
dup clone freeze itself
|
|
@@ -100,7 +100,7 @@ module Mutineer
|
|
|
100
100
|
link = top.receiver
|
|
101
101
|
while dotted?(link)
|
|
102
102
|
@links[link] = true
|
|
103
|
-
|
|
103
|
+
SKIPPED.include?(link.name) ? decline : drop(link)
|
|
104
104
|
link = link.receiver
|
|
105
105
|
end
|
|
106
106
|
end
|
|
@@ -22,7 +22,8 @@ module Mutineer
|
|
|
22
22
|
# as the `nil` that `statement_removal` or `return_nil` puts in place of the
|
|
23
23
|
# whole conditional. It is still made: dropping it would make the mutants
|
|
24
24
|
# of one operator depend on which other operators run, and on what the user
|
|
25
|
-
# suppressed for them. The two then get the same verdict.
|
|
25
|
+
# suppressed for them. The two then get the same verdict. Their text
|
|
26
|
+
# differs, so the job plan's drop of a repeated edit (#275) keeps both.
|
|
26
27
|
#
|
|
27
28
|
# `condition_negation` wraps the condition in `!( ... )`, which neither
|
|
28
29
|
# forced value is.
|
|
@@ -53,13 +54,15 @@ module Mutineer
|
|
|
53
54
|
private
|
|
54
55
|
|
|
55
56
|
# Emits the forced condition unless the condition is a literal or holds a
|
|
56
|
-
# heredoc, whose body would stay behind as code.
|
|
57
|
+
# heredoc, whose body would stay behind as code. The heredoc skip counts
|
|
58
|
+
# as declined; the literal skip does not.
|
|
57
59
|
#
|
|
58
60
|
# @param node [Prism::IfNode, Prism::UnlessNode] the conditional.
|
|
59
61
|
# @return [void]
|
|
60
62
|
def force(node)
|
|
61
63
|
predicate = node.predicate
|
|
62
|
-
return if LITERALS.include?(unwrap(predicate).class)
|
|
64
|
+
return if LITERALS.include?(unwrap(predicate).class)
|
|
65
|
+
return decline if heredoc?(predicate)
|
|
63
66
|
|
|
64
67
|
loc = predicate.location
|
|
65
68
|
@mutations << Mutation.new(
|
|
@@ -37,13 +37,15 @@ module Mutineer
|
|
|
37
37
|
|
|
38
38
|
# Emits one mutation that keeps the left operand, then one that keeps
|
|
39
39
|
# the right operand. The order is fixed, because the two mutants share
|
|
40
|
-
# a token and their occurrence ordinals tell them apart.
|
|
40
|
+
# a token and their occurrence ordinals tell them apart. A removed
|
|
41
|
+
# operand that holds a heredoc counts as declined; a kept jump does not.
|
|
41
42
|
#
|
|
42
43
|
# @param node [Prism::AndNode, Prism::OrNode] connector node.
|
|
43
44
|
# @return [void]
|
|
44
45
|
def emit(node)
|
|
45
46
|
[[node.left, node.right], [node.right, node.left]].each do |kept, removed|
|
|
46
|
-
next if JUMPS.any? { |type| kept.is_a?(type) }
|
|
47
|
+
next if JUMPS.any? { |type| kept.is_a?(type) }
|
|
48
|
+
next decline if heredoc?(removed)
|
|
47
49
|
|
|
48
50
|
@mutations << Mutation.new(
|
|
49
51
|
start_offset: node.location.start_offset,
|