mutineer 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +466 -0
- data/README.md +157 -747
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/cli.rb +132 -28
- data/lib/mutineer/concern_blocks.rb +187 -5
- data/lib/mutineer/config.rb +6 -0
- data/lib/mutineer/coverage_map.rb +92 -17
- data/lib/mutineer/daemon_backend.rb +75 -32
- data/lib/mutineer/daemon_client.rb +50 -17
- data/lib/mutineer/daemon_server.rb +93 -41
- data/lib/mutineer/external_backend.rb +150 -17
- data/lib/mutineer/init.rb +1 -1
- data/lib/mutineer/isolation.rb +70 -25
- data/lib/mutineer/job_plan.rb +119 -34
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +10 -8
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +22 -0
- data/lib/mutineer/mutators/call_to_receiver.rb +20 -7
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +18 -6
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +226 -12
- data/lib/mutineer/rails_worker_db.rb +200 -16
- data/lib/mutineer/redaction.rb +112 -11
- data/lib/mutineer/reporter.rb +109 -21
- data/lib/mutineer/result.rb +33 -5
- data/lib/mutineer/runner.rb +126 -18
- data/lib/mutineer/statement_lines.rb +14 -8
- data/lib/mutineer/subject.rb +13 -5
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners.rb +4 -2
- data/lib/mutineer/triage.rb +2 -2
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/skills/mutineer/SKILL.md +84 -0
- metadata +5 -1
data/lib/mutineer/isolation.rb
CHANGED
|
@@ -7,6 +7,7 @@ require_relative "child_stdout"
|
|
|
7
7
|
require_relative "child_coverage"
|
|
8
8
|
require_relative "kill_channel"
|
|
9
9
|
require_relative "concern_blocks"
|
|
10
|
+
require_relative "redaction"
|
|
10
11
|
|
|
11
12
|
module Mutineer
|
|
12
13
|
# Fork-based isolation for running one mutant. The block runs in a child
|
|
@@ -66,7 +67,7 @@ module Mutineer
|
|
|
66
67
|
code = e.status
|
|
67
68
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
68
69
|
# STDERR, not `warn`: a test may have left `$stderr` as a StringIO.
|
|
69
|
-
STDERR.puts "[mutineer-child] #{
|
|
70
|
+
STDERR.puts "[mutineer-child] #{Redaction.describe(e)}"
|
|
70
71
|
code = 2
|
|
71
72
|
end
|
|
72
73
|
STDERR.flush
|
|
@@ -138,9 +139,10 @@ module Mutineer
|
|
|
138
139
|
#
|
|
139
140
|
# The verdict matches a run without `--matrix`, which skips everything after
|
|
140
141
|
# the first failing serial test except the code around it (see
|
|
141
|
-
# {KillChannel}). So when a serial test has named a
|
|
142
|
+
# {KillChannel}). So when a serial test has named a failure and the child then
|
|
142
143
|
# ended inside a region that run would skip (an exit, a crash or the
|
|
143
|
-
# timeout in a later test, class or group),
|
|
144
|
+
# timeout in a later test, class or group), its first failure decides:
|
|
145
|
+
# `error` for an unavailable database outcome, otherwise `killed`. The
|
|
144
146
|
# plain run never reached that point. An end anywhere else keeps the exit
|
|
145
147
|
# status, because the plain run reaches that code too: the rest of the
|
|
146
148
|
# failing test's class wrapper or group hooks, or the suite hooks. A kill in
|
|
@@ -167,6 +169,7 @@ module Mutineer
|
|
|
167
169
|
return result.with(kills: Kills.new(killed_by: [], ran: [], complete: false)) if report.invalid
|
|
168
170
|
|
|
169
171
|
result = Result.killed if report.started && report.serial_kill && report.skipping.positive?
|
|
172
|
+
result = Result.error if report.started && report.serial_error && report.skipping.positive?
|
|
170
173
|
agrees = result.killed? ? report.killed.any? : result.survived? && report.killed.empty?
|
|
171
174
|
complete = finished && report.started && report.finished && report.lost.zero? && agrees
|
|
172
175
|
result.with(kills: Kills.new(killed_by: report.killed, ran: report.ran, complete: complete))
|
|
@@ -189,17 +192,46 @@ module Mutineer
|
|
|
189
192
|
#
|
|
190
193
|
# A concern's stored `included` and `prepended` blocks from the source file
|
|
191
194
|
# are forgotten around the load, so ActiveSupport accepts the blocks the
|
|
192
|
-
# temp file stores (#256).
|
|
195
|
+
# temp file stores (#256). For a def in such a block, the caller passes the
|
|
196
|
+
# text from {JobPlan.hook_mutant} and then calls
|
|
197
|
+
# {ConcernBlocks.place_hook_def} (#297).
|
|
198
|
+
#
|
|
199
|
+
# {ConcernBlocks.load_mutant} turns Ruby's warnings off for the load, so
|
|
200
|
+
# the constants it assigns again do not warn (#364).
|
|
193
201
|
#
|
|
194
202
|
# @api private
|
|
195
203
|
# @param mutated [String] mutated source text.
|
|
196
204
|
# @param source_file [String] original source file path.
|
|
197
205
|
# @return [Object] whatever `load` returns.
|
|
198
206
|
def self.apply_whole_file(mutated, source_file)
|
|
199
|
-
|
|
200
|
-
|
|
207
|
+
write_beside(mutated, source_file) { |path| ConcernBlocks.load_mutant(source_file, path) }
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Writes `code` to a hidden `.mutineer_mutant` file beside `source_file`,
|
|
211
|
+
# yields its path, and removes it after the block. Both strategies write
|
|
212
|
+
# through here. Only redefine enables the system-temp fallback when the
|
|
213
|
+
# source directory is not writable.
|
|
214
|
+
#
|
|
215
|
+
# The file is locked until the block ends, as the daemon locks its own file
|
|
216
|
+
# (#295). A SIGKILL drops the lock too. The pid in the name and the lock
|
|
217
|
+
# tell another run's sweep that the file is in use, so the sweep does not
|
|
218
|
+
# delete it between the write and the `load`.
|
|
219
|
+
#
|
|
220
|
+
# @api private
|
|
221
|
+
# @param code [String] the Ruby source to write.
|
|
222
|
+
# @param source_file [String] the source file the code replaces.
|
|
223
|
+
# @param fallback [Boolean] use the system temp directory when the source
|
|
224
|
+
# directory is not writable; redefine only.
|
|
225
|
+
# @yieldparam path [String] absolute path of the written file.
|
|
226
|
+
# @return [Object] whatever the block returns.
|
|
227
|
+
def self.write_beside(code, source_file, fallback: false)
|
|
228
|
+
dir = File.dirname(File.expand_path(source_file))
|
|
229
|
+
dir = Dir.tmpdir if fallback && !File.writable?(dir)
|
|
230
|
+
Tempfile.create([".mutineer_mutant", ".rb"], dir) do |f|
|
|
231
|
+
f.flock(File::LOCK_EX) rescue nil # rubocop:disable Style/RescueModifier
|
|
232
|
+
f.write(code)
|
|
201
233
|
f.flush
|
|
202
|
-
|
|
234
|
+
yield f.path
|
|
203
235
|
end
|
|
204
236
|
end
|
|
205
237
|
|
|
@@ -218,13 +250,6 @@ module Mutineer
|
|
|
218
250
|
# @param source [String] full source text.
|
|
219
251
|
# @return [Object] whatever `load` returns.
|
|
220
252
|
def self.apply_surgical(mutation, subject, source)
|
|
221
|
-
loc = subject.def_node.location
|
|
222
|
-
def_start = loc.start_offset
|
|
223
|
-
snippet = source.byteslice(def_start...loc.end_offset)
|
|
224
|
-
rel_s = mutation.start_offset - def_start
|
|
225
|
-
rel_e = mutation.end_offset - def_start
|
|
226
|
-
mutated_def = snippet.byteslice(0...rel_s) + mutation.replacement + snippet.byteslice(rel_e..)
|
|
227
|
-
|
|
228
253
|
# Rebuild the FULL namespace nesting textually so unqualified enclosing-
|
|
229
254
|
# namespace constants resolve exactly as the reload strategy would. A
|
|
230
255
|
# bare redefinition on the owner would collapse Module.nesting to [owner]
|
|
@@ -240,12 +265,7 @@ module Mutineer
|
|
|
240
265
|
# falsely survives. Re-open the singleton class so the redefinition lands on
|
|
241
266
|
# the same method the test calls. `def self.foo` already carries its
|
|
242
267
|
# receiver, so it is left as-is (wrapping it would mis-target).
|
|
243
|
-
inner =
|
|
244
|
-
if subject.singleton && subject.def_node.receiver.nil?
|
|
245
|
-
"class << self\n#{mutated_def}\nend"
|
|
246
|
-
else
|
|
247
|
-
mutated_def
|
|
248
|
-
end
|
|
268
|
+
inner = mutated_def(mutation, subject, source)
|
|
249
269
|
inner = "#{subject.block_owner}.class_eval do\n#{inner}\nend" if subject.block_owner
|
|
250
270
|
wrapped = "#{prefix}#{inner}#{"\nend" * keywords.size}"
|
|
251
271
|
|
|
@@ -267,15 +287,40 @@ module Mutineer
|
|
|
267
287
|
# Module.nesting identically, with no dynamic string execution for
|
|
268
288
|
# scanners to flag. The input is the project's OWN source (the enclosing
|
|
269
289
|
# method, textually mutated), loaded only in this forked child.
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
290
|
+
#
|
|
291
|
+
# The file sits beside the source, as in apply_whole_file, so `__dir__`
|
|
292
|
+
# and `require_relative` in the method resolve against its real
|
|
293
|
+
# neighbours (#279). In the temp dir they raise ENOENT or LoadError, and
|
|
294
|
+
# every mutant of the method dies whatever it changes. The hidden
|
|
295
|
+
# `.mutineer_mutant` name keeps Zeitwerk away (#253) and lets the parent
|
|
296
|
+
# sweep an orphan after a timeout kill.
|
|
297
|
+
#
|
|
298
|
+
# A read-only source dir keeps the 1.x system-temp location, so redefine
|
|
299
|
+
# still runs there (only path-relative methods mis-score, as before: #339).
|
|
300
|
+
write_beside(wrapped, subject.file, fallback: true) { |path| load path }
|
|
275
301
|
|
|
276
302
|
target.send(vis, subject.name) if vis && vis != :public
|
|
277
303
|
end
|
|
278
304
|
|
|
305
|
+
# The enclosing def with the mutation applied, wrapped in `class << self`
|
|
306
|
+
# for a singleton def with no receiver (#20). Pure text: it reads no live
|
|
307
|
+
# module, so the tool process can call it too ({JobPlan.hook_mutant}).
|
|
308
|
+
#
|
|
309
|
+
# @api private
|
|
310
|
+
# @param mutation [Mutineer::Mutation] mutation to apply.
|
|
311
|
+
# @param subject [Mutineer::Subject] subject being mutated.
|
|
312
|
+
# @param source [String] full source text.
|
|
313
|
+
# @return [String] the mutated def.
|
|
314
|
+
def self.mutated_def(mutation, subject, source)
|
|
315
|
+
loc = subject.def_node.location
|
|
316
|
+
def_start = loc.start_offset
|
|
317
|
+
snippet = source.byteslice(def_start...loc.end_offset)
|
|
318
|
+
rel_s = mutation.start_offset - def_start
|
|
319
|
+
rel_e = mutation.end_offset - def_start
|
|
320
|
+
mutated = snippet.byteslice(0...rel_s) + mutation.replacement + snippet.byteslice(rel_e..)
|
|
321
|
+
subject.singleton && subject.def_node.receiver.nil? ? "class << self\n#{mutated}\nend" : mutated
|
|
322
|
+
end
|
|
323
|
+
|
|
279
324
|
# Resolve each namespace ELEMENT to its live Module and pick the correct
|
|
280
325
|
# keyword (reopening a class with `module` — or vice versa — raises
|
|
281
326
|
# TypeError), so the textual wrapper matches the real definitions.
|
data/lib/mutineer/job_plan.rb
CHANGED
|
@@ -11,6 +11,8 @@ require_relative "mutator_registry"
|
|
|
11
11
|
require_relative "mutant_id"
|
|
12
12
|
require_relative "project_path"
|
|
13
13
|
require_relative "external_backend"
|
|
14
|
+
require_relative "isolation"
|
|
15
|
+
require_relative "test_runners"
|
|
14
16
|
require_relative "orphan_guard"
|
|
15
17
|
|
|
16
18
|
module Mutineer
|
|
@@ -37,7 +39,20 @@ module Mutineer
|
|
|
37
39
|
# `{id:, file:, subject:}` hash per matched mutant, in collection order: its
|
|
38
40
|
# new id, its project-relative file and its subject's qualified name;
|
|
39
41
|
# recorded even when a new id is also listed, since the old entry still
|
|
40
|
-
# over-matches other files)
|
|
42
|
+
# over-matches other files), `id_map` (every reported mutant's new id =>
|
|
43
|
+
# its old-format id), `dropped_id_map` (the same for each dropped repeat of
|
|
44
|
+
# an edit: its id is still current for `mutineer migrate`) and `declined`
|
|
45
|
+
# (operator name => count of candidates that operator found and did not
|
|
46
|
+
# mutate, {Mutators::Base#declined}; only counts above zero, keys sorted).
|
|
47
|
+
# The name is the {MutatorRegistry::ALL} key, or the class's `to_s` for an
|
|
48
|
+
# operator outside the registry. `--only` narrows `declined`, and `--since`
|
|
49
|
+
# does not: a declined candidate has no mutation to place on a line.
|
|
50
|
+
#
|
|
51
|
+
# Mutations are collected in {MutatorRegistry::ALL} order, whatever the
|
|
52
|
+
# order of `operator_classes`. Two operators can give the same mutated
|
|
53
|
+
# source (#275), and the copy of the registry-first operator is the one
|
|
54
|
+
# that runs, so the kept copy and the run order do not depend on the
|
|
55
|
+
# `--operators` order.
|
|
41
56
|
#
|
|
42
57
|
# Subjects sharing a qualified name in one file (two owner-less `def index`
|
|
43
58
|
# in two DSL blocks) get a per-file ordinal in discovery order, so their ids
|
|
@@ -46,8 +61,15 @@ module Mutineer
|
|
|
46
61
|
# @param config [Mutineer::Config] run configuration.
|
|
47
62
|
# @param operator_classes [Array<Class>] resolved operators.
|
|
48
63
|
# @return [Array(Array, Array<Result>, Hash<String,String>, Hash{Symbol => Hash})]
|
|
49
|
-
# jobs, ignored, source_map, and extras (`:legacy_ignore_matches`,
|
|
64
|
+
# jobs, ignored, source_map, and extras (`:legacy_ignore_matches`,
|
|
65
|
+
# `:id_map`, `:dropped_id_map`, `:declined`).
|
|
50
66
|
def self.collect_jobs(config, operator_classes)
|
|
67
|
+
# Registry order, so a repeat across operators keeps the registry-first
|
|
68
|
+
# copy (#275). A class outside ALL goes last; ties keep the input order.
|
|
69
|
+
rank = MutatorRegistry::ALL.values
|
|
70
|
+
operator_classes = operator_classes.each_with_index
|
|
71
|
+
.sort_by { |klass, i| [rank.index(klass) || rank.size, i] }
|
|
72
|
+
.map(&:first)
|
|
51
73
|
source_map = {}
|
|
52
74
|
disabled_map = {}
|
|
53
75
|
id_paths = {}
|
|
@@ -61,10 +83,18 @@ module Mutineer
|
|
|
61
83
|
ignored_results = []
|
|
62
84
|
legacy_ignore_matches = {}
|
|
63
85
|
id_map = {}
|
|
86
|
+
dropped_id_map = {}
|
|
87
|
+
declined = Hash.new(0)
|
|
64
88
|
Project.discover(config.sources, only: config.only).each do |subject|
|
|
65
|
-
source = (source_map[subject.file] ||= File.read(subject.file))
|
|
89
|
+
source = (source_map[subject.file] ||= File.read(subject.file, encoding: "UTF-8"))
|
|
66
90
|
disabled = (disabled_map[subject.file] ||= suppress_map(source, subject.file))
|
|
67
|
-
|
|
91
|
+
# A repeated operator name runs once, so its declines count once.
|
|
92
|
+
mutations = operator_classes.uniq.flat_map do |klass|
|
|
93
|
+
operator = klass.new
|
|
94
|
+
found = operator.mutations_for(subject, source)
|
|
95
|
+
declined[MutatorRegistry::ALL.key(klass) || klass.to_s] += operator.declined
|
|
96
|
+
found
|
|
97
|
+
end
|
|
68
98
|
id_path = (id_paths[subject.file] ||= ProjectPath.relative(subject.file, config.project_root))
|
|
69
99
|
decls = name_decls[[id_path, subject.qualified_name]]
|
|
70
100
|
ordinal = (decls[subject.def_node.location.start_offset] ||= decls.size)
|
|
@@ -72,22 +102,27 @@ module Mutineer
|
|
|
72
102
|
legacy_ids = MutantId.legacy_for_subject(subject, source, mutations)
|
|
73
103
|
lines = mutations.map { |m| source.byteslice(0, m.start_offset).count("\n") + 1 }
|
|
74
104
|
keys = result_keys(mutations, source, lines)
|
|
75
|
-
# A repeat of an earlier edit on the same line
|
|
76
|
-
# separately among the run and the ignored
|
|
77
|
-
# never hides a copy that should run. A
|
|
105
|
+
# A repeat of an earlier edit on the same line, from any operator, is
|
|
106
|
+
# dropped (#159, #275), separately among the run and the ignored
|
|
107
|
+
# mutants, so an ignored copy never hides a copy that should run. A
|
|
108
|
+
# dropped copy is not reported, but its id stays current for migrate.
|
|
78
109
|
seen = { run: Set.new, ignored: Set.new }
|
|
79
110
|
mutations.each_with_index do |mutation, i|
|
|
80
111
|
id = ids[i]
|
|
81
112
|
legacy = legacy_ids[i]
|
|
82
113
|
ignored = suppressed?(mutation.operator, lines[i], [id, legacy], disabled, ignore_set)
|
|
83
|
-
next unless seen[ignored ? :ignored : :run].add?(keys[i])
|
|
84
|
-
|
|
85
|
-
id_map[id] = legacy
|
|
86
114
|
# An old entry still over-matches other files even when the new id is
|
|
87
|
-
# listed too, so every old-entry match is reported for migration
|
|
115
|
+
# listed too, so every old-entry match is reported for migration,
|
|
116
|
+
# including a dropped copy's.
|
|
88
117
|
if ignore_set.include?(legacy)
|
|
89
118
|
(legacy_ignore_matches[legacy] ||= []) << { id: id, file: id_path, subject: subject.qualified_name }
|
|
90
119
|
end
|
|
120
|
+
unless seen[ignored ? :ignored : :run].add?(keys[i])
|
|
121
|
+
dropped_id_map[id] = legacy
|
|
122
|
+
next
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
id_map[id] = legacy
|
|
91
126
|
if ignored
|
|
92
127
|
reason = suppression_reason(mutation.operator, lines[i], [id, legacy], disabled, ignore_reasons)
|
|
93
128
|
ignored_results << Result.ignored.with(subject: subject, mutation: mutation, id: id, reason: reason)
|
|
@@ -96,13 +131,15 @@ module Mutineer
|
|
|
96
131
|
end
|
|
97
132
|
end
|
|
98
133
|
end
|
|
99
|
-
|
|
134
|
+
declined = declined.select { |_, count| count.positive? }.sort.to_h
|
|
135
|
+
[jobs, ignored_results, source_map, { legacy_ignore_matches: legacy_ignore_matches, id_map: id_map,
|
|
136
|
+
dropped_id_map: dropped_id_map, declined: declined }]
|
|
100
137
|
end
|
|
101
138
|
|
|
102
|
-
# One key per mutation; two mutations share a key exactly when they are
|
|
103
|
-
# the same
|
|
104
|
-
#
|
|
105
|
-
# group of same-
|
|
139
|
+
# One key per mutation; two mutations share a key exactly when they are on
|
|
140
|
+
# the same line and give the same mutated source (one edit), from one
|
|
141
|
+
# operator (#159) or two (#275). The caller drops a repeat after ids are
|
|
142
|
+
# assigned. Only a group of same-line mutations can repeat, so only those
|
|
106
143
|
# build a key from their text: the span from the group's earliest start to
|
|
107
144
|
# its latest end, digested. Every other mutation gets a key of its own.
|
|
108
145
|
#
|
|
@@ -112,7 +149,7 @@ module Mutineer
|
|
|
112
149
|
# @return [Array<Object>] one key per mutation, in order.
|
|
113
150
|
def self.result_keys(mutations, source, lines)
|
|
114
151
|
keys = Array.new(mutations.size) { |i| i }
|
|
115
|
-
mutations.each_index.group_by { |i|
|
|
152
|
+
mutations.each_index.group_by { |i| lines[i] }.each_value do |group|
|
|
116
153
|
next if group.size < 2
|
|
117
154
|
|
|
118
155
|
from = group.map { |i| mutations[i].start_offset }.min
|
|
@@ -120,7 +157,7 @@ module Mutineer
|
|
|
120
157
|
group.each do |i|
|
|
121
158
|
m = mutations[i]
|
|
122
159
|
span = "#{source.byteslice(from...m.start_offset)}#{m.replacement}#{source.byteslice(m.end_offset...to)}"
|
|
123
|
-
keys[i] = [
|
|
160
|
+
keys[i] = [lines[i], Digest::SHA256.digest(span)]
|
|
124
161
|
end
|
|
125
162
|
end
|
|
126
163
|
keys
|
|
@@ -130,10 +167,14 @@ module Mutineer
|
|
|
130
167
|
# those results would treat existing assertion failures as killed mutants.
|
|
131
168
|
#
|
|
132
169
|
# @param coverage_map [Mutineer::CoverageMap] the built or loaded map.
|
|
170
|
+
# @param framework [String, nil] the run's framework (nil is Minitest, as in
|
|
171
|
+
# {TestRunners.for}). For Minitest, the message names the seed: an
|
|
172
|
+
# order-dependent suite can pass in a random order and fail in the
|
|
173
|
+
# pinned one (#282).
|
|
133
174
|
# @return [void]
|
|
134
175
|
# @raise [Mutineer::SmokeCheckError] when any captured test failed clean,
|
|
135
176
|
# or when no test recorded coverage and a capture failed.
|
|
136
|
-
def self.abort_if_unclean!(coverage_map)
|
|
177
|
+
def self.abort_if_unclean!(coverage_map, framework: nil)
|
|
137
178
|
if coverage_map.map.empty? && coverage_map.failed_test_files.any?
|
|
138
179
|
raise SmokeCheckError, "no test recorded coverage, and capture failed for #{coverage_map.failed_test_files.join(', ')}"
|
|
139
180
|
end
|
|
@@ -141,9 +182,48 @@ module Mutineer
|
|
|
141
182
|
files = coverage_map.failed_clean_tests
|
|
142
183
|
return if files.empty?
|
|
143
184
|
|
|
185
|
+
seed_hint = TestRunners.for(framework) == TestRunners::Minitest ? minitest_seed_hint : ""
|
|
186
|
+
# The hint opens with "the unmutated suite is not green", so the files
|
|
187
|
+
# follow it rather than repeat that sentence (#360).
|
|
144
188
|
raise SmokeCheckError,
|
|
145
|
-
"
|
|
146
|
-
|
|
189
|
+
"#{ExternalBackend.generic_env_hint}. Failed: #{files.join(', ')}.#{seed_hint}"
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Names the configured seed, and accounts for helper overrides.
|
|
193
|
+
#
|
|
194
|
+
# @api private
|
|
195
|
+
# @return [String]
|
|
196
|
+
def self.minitest_seed_hint
|
|
197
|
+
seed = MinitestIntegration.seed
|
|
198
|
+
" Mutineer runs Minitest with SEED=#{seed}, unless a test helper changes SEED. " \
|
|
199
|
+
"An order-dependent test can fail here and pass in another order. " \
|
|
200
|
+
"Check the listed test files with the same seed and helpers " \
|
|
201
|
+
"(for \"combined suite\", all of them together). " \
|
|
202
|
+
"Change SEED, including helper settings, to choose another order."
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# The reload mutant text for a def in a concern's `included` or `prepended`
|
|
206
|
+
# block (#297): the mutated file with one statement added just before the
|
|
207
|
+
# hook call, `::Mutineer::ConcernBlocks.store_hook_def do <mutated def> end`.
|
|
208
|
+
# The statement sits in the same body as the hook call, so the stored def has
|
|
209
|
+
# the block's constant scope. After the load, the child passes the hook-def
|
|
210
|
+
# values to {ConcernBlocks.place_hook_def}. Shared by {Runner.run} and
|
|
211
|
+
# {DaemonBackend.job_result}, so both build the same text. The hook call starts
|
|
212
|
+
# before the def, so the mutation does not move its offset. A bad insert is
|
|
213
|
+
# not checked here: the child's `load` raises `SyntaxError` before any test
|
|
214
|
+
# runs, so the mutant scores `error`.
|
|
215
|
+
#
|
|
216
|
+
# @param mutated [String] the mutated source text.
|
|
217
|
+
# @param mutation [Mutineer::Mutation] the mutation.
|
|
218
|
+
# @param subject [Mutineer::Subject] a subject with a {Subject#reload_hook_call}.
|
|
219
|
+
# @param source [String] the original source text.
|
|
220
|
+
# @return [Array(String, Hash{Symbol => Object})] the mutant text, and the
|
|
221
|
+
# original def's `line`, `name` and `singleton`.
|
|
222
|
+
def self.hook_mutant(mutated, mutation, subject, source)
|
|
223
|
+
at = subject.reload_hook_call.location.start_offset
|
|
224
|
+
stored = "::Mutineer::ConcernBlocks.store_hook_def do\n#{Isolation.mutated_def(mutation, subject, source)}\nend; "
|
|
225
|
+
[mutated.byteslice(0...at) + stored + mutated.byteslice(at..),
|
|
226
|
+
{ line: subject.def_node.location.start_line, name: subject.name, singleton: subject.singleton }]
|
|
147
227
|
end
|
|
148
228
|
|
|
149
229
|
# Coverage-based test selection, shared by the in-process ({Runner.run}) and daemon
|
|
@@ -152,11 +232,14 @@ module Mutineer
|
|
|
152
232
|
# order of {CoverageMap#order_tests} (paired files, then cheapest), or
|
|
153
233
|
# `[:verdict, Result]` (no_coverage / uncapturable) when none do. A line
|
|
154
234
|
# Ruby does not count (a later line of a multi-line statement) uses the
|
|
155
|
-
# tests that ran the statement that holds it ({StatementLines}). When
|
|
156
|
-
#
|
|
157
|
-
#
|
|
158
|
-
#
|
|
159
|
-
#
|
|
235
|
+
# tests that ran the statement that holds it ({StatementLines}). When those
|
|
236
|
+
# lines give no tests, it retries with the loose lines of the statement:
|
|
237
|
+
# code of the statement that runs only sometimes (the right side of a
|
|
238
|
+
# multi-line `a && b`, #254), and a first or last line that holds other
|
|
239
|
+
# code (the opener of a one-line block, #289), but never the `def` line.
|
|
240
|
+
# Those tests may not have run the mutant, so the answer is
|
|
241
|
+
# `[:maybe, abs_test_paths]` and {selection_verdict} treats a survivor as
|
|
242
|
+
# no_coverage.
|
|
160
243
|
#
|
|
161
244
|
# An empty selection is `:ran_at_load` when the mutant's line ran while the
|
|
162
245
|
# app booted or its class loaded ({ran_at_load?}): no test ran it, but the
|
|
@@ -182,11 +265,12 @@ module Mutineer
|
|
|
182
265
|
if chosen.empty? && subject
|
|
183
266
|
# A multi-line statement has a count on one of its lines only.
|
|
184
267
|
lines = StatementLines.for(subject.def_node, source, mutation.start_offset)
|
|
185
|
-
|
|
268
|
+
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
269
|
+
if chosen.empty?
|
|
186
270
|
lines = StatementLines.for(subject.def_node, source, mutation.start_offset, loose: true)
|
|
271
|
+
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
187
272
|
kind = :maybe
|
|
188
273
|
end
|
|
189
|
-
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
190
274
|
end
|
|
191
275
|
if chosen.empty?
|
|
192
276
|
# Method BODY range, not the whole def: the def/end lines are "covered" at
|
|
@@ -435,16 +519,17 @@ module Mutineer
|
|
|
435
519
|
config.sources.map { |f| File.dirname(File.expand_path(f, config.project_root)) }.uniq
|
|
436
520
|
end
|
|
437
521
|
|
|
438
|
-
# Removes stale mutant tempfiles from the given directories. The
|
|
439
|
-
#
|
|
440
|
-
#
|
|
441
|
-
#
|
|
442
|
-
#
|
|
522
|
+
# Removes stale mutant tempfiles from the given directories. The default
|
|
523
|
+
# glob matches the files of both modes, so the in-process boot sweep also
|
|
524
|
+
# removes an old daemon file (#287). {DaemonBackend} passes its daemon-only
|
|
525
|
+
# glob when it sweeps tool-side at the end of every daemon run, empty or not.
|
|
526
|
+
# A file another process still owns is left in place: both modes record the
|
|
527
|
+
# creating pid in the name and lock the file while it loads (#295).
|
|
443
528
|
#
|
|
444
529
|
# @param dirs [Array<String>] directories to sweep.
|
|
445
530
|
# @param glob [String] filename pattern to remove.
|
|
446
531
|
# @return [Array<String>] the files left in place because a process owns them.
|
|
447
|
-
def self.sweep_orphans(dirs, glob =
|
|
532
|
+
def self.sweep_orphans(dirs, glob = OrphanGuard::TEMP_BASENAME)
|
|
448
533
|
dirs.flat_map do |dir|
|
|
449
534
|
Dir.glob(File.join(dir, glob)).select do |path|
|
|
450
535
|
next true if OrphanGuard.mutant_file_in_use?(path)
|
|
@@ -9,6 +9,7 @@ module Mutineer
|
|
|
9
9
|
# ["start"] the recorder is armed; once, first
|
|
10
10
|
# ["pass", file, name, id] a test passed against the mutant
|
|
11
11
|
# ["kill", file, name, id] a test failed or errored: it killed it
|
|
12
|
+
# ["unavailable", file, name, id] a test ran but its database failed
|
|
12
13
|
# ["parallel"] the run reached its parallel tests; once
|
|
13
14
|
# ["skip"] what runs next, a run without --matrix
|
|
14
15
|
# skips (see below)
|
|
@@ -27,20 +28,22 @@ module Mutineer
|
|
|
27
28
|
#
|
|
28
29
|
# Minitest runs its serial test classes first and its `parallelize_me!`
|
|
29
30
|
# classes after them, and the stop at the first failure of a run without
|
|
30
|
-
# `--matrix` skips everything after a serial
|
|
31
|
+
# `--matrix` skips everything after a serial failure. `parallel` marks where the
|
|
31
32
|
# parallel tests begin, so the parent can tell a kill that the stop would have
|
|
32
33
|
# followed from one it could not. `cleanup` marks where the tests end and the
|
|
33
34
|
# suite's own hooks begin (RSpec `after(:suite)`). It decides no verdict
|
|
34
35
|
# (the `skip` regions do); it only marks a later test line as out of order.
|
|
35
36
|
#
|
|
36
|
-
# After the first serial
|
|
37
|
+
# After the first serial failure, a run without `--matrix` skips every later
|
|
37
38
|
# test and every later test class or example group, but it still runs the
|
|
38
39
|
# rest of the code around the failing test: the remainder of its Minitest
|
|
39
40
|
# class `run` wrapper, the `after(:all)` hooks of its RSpec groups, and the
|
|
40
41
|
# suite hooks. The recorder writes `skip` as it enters something that run
|
|
41
42
|
# would skip and `unskip` as that returns, so a child that exits, crashes or
|
|
42
43
|
# hangs inside a skipped region is one the plain run would have stopped
|
|
43
|
-
# before (see {Isolation.finish}).
|
|
44
|
+
# before (see {Isolation.finish}). Its first failure decides the verdict:
|
|
45
|
+
# killed for a real kill, error for an unavailable database outcome. The
|
|
46
|
+
# pairs nest. An unavailable result also leaves the row incomplete.
|
|
44
47
|
#
|
|
45
48
|
# A row is complete only with `start` and `end`, in order, and no lost line: a
|
|
46
49
|
# test that exits the process, a crash, an interrupted run or a recorder that
|
|
@@ -61,6 +64,9 @@ module Mutineer
|
|
|
61
64
|
# Event for a test that failed or errored against the mutant: it killed it.
|
|
62
65
|
KILL = "kill"
|
|
63
66
|
|
|
67
|
+
# Event for a test whose database failed, so it provides no mutation verdict.
|
|
68
|
+
UNAVAILABLE = "unavailable"
|
|
69
|
+
|
|
64
70
|
# Event that opens a recorded run.
|
|
65
71
|
START = "start"
|
|
66
72
|
|
|
@@ -87,17 +93,18 @@ module Mutineer
|
|
|
87
93
|
|
|
88
94
|
# What one child sent. `killed` and `ran` are sorted, unique
|
|
89
95
|
# `[file, name, id]` tests (`ran` includes the killers); `lost` counts lines
|
|
90
|
-
# that could not be read, including a partial last line
|
|
91
|
-
# `cleanup` say the marker arrived
|
|
92
|
-
#
|
|
96
|
+
# that could not be read, including a partial last line, and unavailable
|
|
97
|
+
# outcomes. `parallel` and `cleanup` say the marker arrived. `serial_kill`
|
|
98
|
+
# or `serial_error` says which failure came first before `parallel`.
|
|
99
|
+
# `skipping` counts the {SKIP} regions still
|
|
93
100
|
# open when the stream ended. `invalid` is set by any line out of order.
|
|
94
|
-
Report = Struct.new(:killed, :ran, :started, :finished, :parallel, :cleanup, :serial_kill, :skipping, :invalid,
|
|
101
|
+
Report = Struct.new(:killed, :ran, :started, :finished, :parallel, :cleanup, :serial_kill, :serial_error, :skipping, :invalid,
|
|
95
102
|
:lost, keyword_init: true)
|
|
96
103
|
|
|
97
104
|
# Writes one test line.
|
|
98
105
|
#
|
|
99
106
|
# @param io [IO] the write end of the channel.
|
|
100
|
-
# @param event [String] {PASS} or {
|
|
107
|
+
# @param event [String] {PASS}, {KILL}, or {UNAVAILABLE}.
|
|
101
108
|
# @param file [String] the file that defines the test.
|
|
102
109
|
# @param name [String] the test's display name, e.g. `CalculatorTest#test_add`.
|
|
103
110
|
# @param id [String] what tells the test apart from others with its file and
|
|
@@ -169,7 +176,7 @@ module Mutineer
|
|
|
169
176
|
# @return [Report]
|
|
170
177
|
def self.parse(buffer)
|
|
171
178
|
report = Report.new(killed: [], ran: [], started: false, finished: false, parallel: false, cleanup: false,
|
|
172
|
-
serial_kill: false, skipping: 0, invalid: false, lost: 0)
|
|
179
|
+
serial_kill: false, serial_error: false, skipping: 0, invalid: false, lost: 0)
|
|
173
180
|
buffer.dup.force_encoding(Encoding::UTF_8).each_line do |line|
|
|
174
181
|
fields = line.end_with?("\n") ? parse_line(line) : nil
|
|
175
182
|
fields ? apply(report, fields) : report.lost += 1
|
|
@@ -211,10 +218,13 @@ module Mutineer
|
|
|
211
218
|
report.invalid = true unless open?(report) && !report.cleanup
|
|
212
219
|
test = fields[1, 3]
|
|
213
220
|
report.ran << test
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
221
|
+
if fields[0] == UNAVAILABLE
|
|
222
|
+
report.lost += 1
|
|
223
|
+
report.serial_error = true unless report.parallel || report.serial_kill
|
|
224
|
+
elsif fields[0] == KILL
|
|
225
|
+
report.killed << test
|
|
226
|
+
report.serial_kill = true unless report.parallel || report.serial_error
|
|
227
|
+
end
|
|
218
228
|
end
|
|
219
229
|
end
|
|
220
230
|
|
|
@@ -238,7 +248,7 @@ module Mutineer
|
|
|
238
248
|
|
|
239
249
|
valid =
|
|
240
250
|
case fields[0]
|
|
241
|
-
when PASS, KILL then fields.size == 4
|
|
251
|
+
when PASS, KILL, UNAVAILABLE then fields.size == 4
|
|
242
252
|
when START, PARALLEL, SKIP, UNSKIP, CLEANUP, LOST, FINISH then fields.size == 1
|
|
243
253
|
end
|
|
244
254
|
fields if valid
|
data/lib/mutineer/migrate.rb
CHANGED
|
@@ -7,11 +7,12 @@ require_relative "mutator_registry"
|
|
|
7
7
|
|
|
8
8
|
module Mutineer
|
|
9
9
|
# Rewrites old-format mutant ids in a `.mutineer.yml` `ignore:` list to the
|
|
10
|
-
# current id format. The map is the inverse of the `id_map`
|
|
11
|
-
# {JobPlan.collect_jobs}: each old id goes to every new
|
|
12
|
-
# to it,
|
|
13
|
-
#
|
|
14
|
-
#
|
|
10
|
+
# current id format. The map is the inverse of the `id_map` and
|
|
11
|
+
# `dropped_id_map` from {JobPlan.collect_jobs}: each old id goes to every new
|
|
12
|
+
# id that still hashes to it, reported ids first, each group in collection
|
|
13
|
+
# order. A dropped repeat of an edit is not reported, but its id is still
|
|
14
|
+
# current (#275). The map covers only the configured operators and the
|
|
15
|
+
# sources on the command line. Ids are static hashes, so this runs no tests.
|
|
15
16
|
#
|
|
16
17
|
# The file is edited as text. A YAML load and dump would drop comments and
|
|
17
18
|
# reorder keys. Only a bare id inside `ignore:` is replaced. A flow list may
|
|
@@ -23,7 +24,7 @@ module Mutineer
|
|
|
23
24
|
# one mapping, so it is left in place and reported for a hand edit. A
|
|
24
25
|
# baseline file is never read.
|
|
25
26
|
module Migrate
|
|
26
|
-
# One old id and the new ids that replace it, in
|
|
27
|
+
# One old id and the new ids that replace it, in {Migrate.id_maps} order.
|
|
27
28
|
Replacement = Data.define(:old_id, :new_ids)
|
|
28
29
|
|
|
29
30
|
# The outcome of rewriting one file in memory. `text` is what to write.
|
|
@@ -42,12 +43,13 @@ module Mutineer
|
|
|
42
43
|
def self.id_maps(config)
|
|
43
44
|
names = config.operators || MutatorRegistry::DEFAULT_NAMES
|
|
44
45
|
_jobs, _ignored, _source_map, extras = JobPlan.collect_jobs(config, MutatorRegistry.resolve(names))
|
|
46
|
+
all = extras[:id_map].merge(extras[:dropped_id_map])
|
|
45
47
|
old_to_new = {}
|
|
46
|
-
|
|
48
|
+
all.each do |new_id, old_id|
|
|
47
49
|
list = (old_to_new[old_id] ||= [])
|
|
48
50
|
list << new_id unless list.include?(new_id)
|
|
49
51
|
end
|
|
50
|
-
[
|
|
52
|
+
[all.keys.to_set, old_to_new]
|
|
51
53
|
end
|
|
52
54
|
|
|
53
55
|
# Rewrites `ignore:` ids in `text`. An id that is already a current new id
|