mutineer 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +432 -0
- data/README.md +157 -747
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/cli.rb +119 -27
- data/lib/mutineer/concern_blocks.rb +168 -5
- data/lib/mutineer/config.rb +6 -0
- data/lib/mutineer/coverage_map.rb +92 -17
- data/lib/mutineer/daemon_backend.rb +75 -32
- data/lib/mutineer/daemon_client.rb +50 -17
- data/lib/mutineer/daemon_server.rb +92 -40
- data/lib/mutineer/external_backend.rb +150 -17
- data/lib/mutineer/isolation.rb +67 -25
- data/lib/mutineer/job_plan.rb +117 -33
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +10 -8
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +22 -0
- data/lib/mutineer/mutators/call_to_receiver.rb +20 -7
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +18 -6
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +226 -12
- data/lib/mutineer/rails_worker_db.rb +200 -16
- data/lib/mutineer/redaction.rb +112 -11
- data/lib/mutineer/reporter.rb +109 -21
- data/lib/mutineer/result.rb +33 -5
- data/lib/mutineer/runner.rb +126 -18
- data/lib/mutineer/statement_lines.rb +14 -8
- data/lib/mutineer/subject.rb +13 -5
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners.rb +4 -2
- data/lib/mutineer/triage.rb +1 -1
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/skills/mutineer/SKILL.md +84 -0
- metadata +5 -1
data/lib/mutineer/job_plan.rb
CHANGED
|
@@ -11,6 +11,8 @@ require_relative "mutator_registry"
|
|
|
11
11
|
require_relative "mutant_id"
|
|
12
12
|
require_relative "project_path"
|
|
13
13
|
require_relative "external_backend"
|
|
14
|
+
require_relative "isolation"
|
|
15
|
+
require_relative "test_runners"
|
|
14
16
|
require_relative "orphan_guard"
|
|
15
17
|
|
|
16
18
|
module Mutineer
|
|
@@ -37,7 +39,20 @@ module Mutineer
|
|
|
37
39
|
# `{id:, file:, subject:}` hash per matched mutant, in collection order: its
|
|
38
40
|
# new id, its project-relative file and its subject's qualified name;
|
|
39
41
|
# recorded even when a new id is also listed, since the old entry still
|
|
40
|
-
# over-matches other files)
|
|
42
|
+
# over-matches other files), `id_map` (every reported mutant's new id =>
|
|
43
|
+
# its old-format id), `dropped_id_map` (the same for each dropped repeat of
|
|
44
|
+
# an edit: its id is still current for `mutineer migrate`) and `declined`
|
|
45
|
+
# (operator name => count of candidates that operator found and did not
|
|
46
|
+
# mutate, {Mutators::Base#declined}; only counts above zero, keys sorted).
|
|
47
|
+
# The name is the {MutatorRegistry::ALL} key, or the class's `to_s` for an
|
|
48
|
+
# operator outside the registry. `--only` narrows `declined`, and `--since`
|
|
49
|
+
# does not: a declined candidate has no mutation to place on a line.
|
|
50
|
+
#
|
|
51
|
+
# Mutations are collected in {MutatorRegistry::ALL} order, whatever the
|
|
52
|
+
# order of `operator_classes`. Two operators can give the same mutated
|
|
53
|
+
# source (#275), and the copy of the registry-first operator is the one
|
|
54
|
+
# that runs, so the kept copy and the run order do not depend on the
|
|
55
|
+
# `--operators` order.
|
|
41
56
|
#
|
|
42
57
|
# Subjects sharing a qualified name in one file (two owner-less `def index`
|
|
43
58
|
# in two DSL blocks) get a per-file ordinal in discovery order, so their ids
|
|
@@ -46,8 +61,15 @@ module Mutineer
|
|
|
46
61
|
# @param config [Mutineer::Config] run configuration.
|
|
47
62
|
# @param operator_classes [Array<Class>] resolved operators.
|
|
48
63
|
# @return [Array(Array, Array<Result>, Hash<String,String>, Hash{Symbol => Hash})]
|
|
49
|
-
# jobs, ignored, source_map, and extras (`:legacy_ignore_matches`,
|
|
64
|
+
# jobs, ignored, source_map, and extras (`:legacy_ignore_matches`,
|
|
65
|
+
# `:id_map`, `:dropped_id_map`, `:declined`).
|
|
50
66
|
def self.collect_jobs(config, operator_classes)
|
|
67
|
+
# Registry order, so a repeat across operators keeps the registry-first
|
|
68
|
+
# copy (#275). A class outside ALL goes last; ties keep the input order.
|
|
69
|
+
rank = MutatorRegistry::ALL.values
|
|
70
|
+
operator_classes = operator_classes.each_with_index
|
|
71
|
+
.sort_by { |klass, i| [rank.index(klass) || rank.size, i] }
|
|
72
|
+
.map(&:first)
|
|
51
73
|
source_map = {}
|
|
52
74
|
disabled_map = {}
|
|
53
75
|
id_paths = {}
|
|
@@ -61,10 +83,18 @@ module Mutineer
|
|
|
61
83
|
ignored_results = []
|
|
62
84
|
legacy_ignore_matches = {}
|
|
63
85
|
id_map = {}
|
|
86
|
+
dropped_id_map = {}
|
|
87
|
+
declined = Hash.new(0)
|
|
64
88
|
Project.discover(config.sources, only: config.only).each do |subject|
|
|
65
|
-
source = (source_map[subject.file] ||= File.read(subject.file))
|
|
89
|
+
source = (source_map[subject.file] ||= File.read(subject.file, encoding: "UTF-8"))
|
|
66
90
|
disabled = (disabled_map[subject.file] ||= suppress_map(source, subject.file))
|
|
67
|
-
|
|
91
|
+
# A repeated operator name runs once, so its declines count once.
|
|
92
|
+
mutations = operator_classes.uniq.flat_map do |klass|
|
|
93
|
+
operator = klass.new
|
|
94
|
+
found = operator.mutations_for(subject, source)
|
|
95
|
+
declined[MutatorRegistry::ALL.key(klass) || klass.to_s] += operator.declined
|
|
96
|
+
found
|
|
97
|
+
end
|
|
68
98
|
id_path = (id_paths[subject.file] ||= ProjectPath.relative(subject.file, config.project_root))
|
|
69
99
|
decls = name_decls[[id_path, subject.qualified_name]]
|
|
70
100
|
ordinal = (decls[subject.def_node.location.start_offset] ||= decls.size)
|
|
@@ -72,22 +102,27 @@ module Mutineer
|
|
|
72
102
|
legacy_ids = MutantId.legacy_for_subject(subject, source, mutations)
|
|
73
103
|
lines = mutations.map { |m| source.byteslice(0, m.start_offset).count("\n") + 1 }
|
|
74
104
|
keys = result_keys(mutations, source, lines)
|
|
75
|
-
# A repeat of an earlier edit on the same line
|
|
76
|
-
# separately among the run and the ignored
|
|
77
|
-
# never hides a copy that should run. A
|
|
105
|
+
# A repeat of an earlier edit on the same line, from any operator, is
|
|
106
|
+
# dropped (#159, #275), separately among the run and the ignored
|
|
107
|
+
# mutants, so an ignored copy never hides a copy that should run. A
|
|
108
|
+
# dropped copy is not reported, but its id stays current for migrate.
|
|
78
109
|
seen = { run: Set.new, ignored: Set.new }
|
|
79
110
|
mutations.each_with_index do |mutation, i|
|
|
80
111
|
id = ids[i]
|
|
81
112
|
legacy = legacy_ids[i]
|
|
82
113
|
ignored = suppressed?(mutation.operator, lines[i], [id, legacy], disabled, ignore_set)
|
|
83
|
-
next unless seen[ignored ? :ignored : :run].add?(keys[i])
|
|
84
|
-
|
|
85
|
-
id_map[id] = legacy
|
|
86
114
|
# An old entry still over-matches other files even when the new id is
|
|
87
|
-
# listed too, so every old-entry match is reported for migration
|
|
115
|
+
# listed too, so every old-entry match is reported for migration,
|
|
116
|
+
# including a dropped copy's.
|
|
88
117
|
if ignore_set.include?(legacy)
|
|
89
118
|
(legacy_ignore_matches[legacy] ||= []) << { id: id, file: id_path, subject: subject.qualified_name }
|
|
90
119
|
end
|
|
120
|
+
unless seen[ignored ? :ignored : :run].add?(keys[i])
|
|
121
|
+
dropped_id_map[id] = legacy
|
|
122
|
+
next
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
id_map[id] = legacy
|
|
91
126
|
if ignored
|
|
92
127
|
reason = suppression_reason(mutation.operator, lines[i], [id, legacy], disabled, ignore_reasons)
|
|
93
128
|
ignored_results << Result.ignored.with(subject: subject, mutation: mutation, id: id, reason: reason)
|
|
@@ -96,13 +131,15 @@ module Mutineer
|
|
|
96
131
|
end
|
|
97
132
|
end
|
|
98
133
|
end
|
|
99
|
-
|
|
134
|
+
declined = declined.select { |_, count| count.positive? }.sort.to_h
|
|
135
|
+
[jobs, ignored_results, source_map, { legacy_ignore_matches: legacy_ignore_matches, id_map: id_map,
|
|
136
|
+
dropped_id_map: dropped_id_map, declined: declined }]
|
|
100
137
|
end
|
|
101
138
|
|
|
102
|
-
# One key per mutation; two mutations share a key exactly when they are
|
|
103
|
-
# the same
|
|
104
|
-
#
|
|
105
|
-
# group of same-
|
|
139
|
+
# One key per mutation; two mutations share a key exactly when they are on
|
|
140
|
+
# the same line and give the same mutated source (one edit), from one
|
|
141
|
+
# operator (#159) or two (#275). The caller drops a repeat after ids are
|
|
142
|
+
# assigned. Only a group of same-line mutations can repeat, so only those
|
|
106
143
|
# build a key from their text: the span from the group's earliest start to
|
|
107
144
|
# its latest end, digested. Every other mutation gets a key of its own.
|
|
108
145
|
#
|
|
@@ -112,7 +149,7 @@ module Mutineer
|
|
|
112
149
|
# @return [Array<Object>] one key per mutation, in order.
|
|
113
150
|
def self.result_keys(mutations, source, lines)
|
|
114
151
|
keys = Array.new(mutations.size) { |i| i }
|
|
115
|
-
mutations.each_index.group_by { |i|
|
|
152
|
+
mutations.each_index.group_by { |i| lines[i] }.each_value do |group|
|
|
116
153
|
next if group.size < 2
|
|
117
154
|
|
|
118
155
|
from = group.map { |i| mutations[i].start_offset }.min
|
|
@@ -120,7 +157,7 @@ module Mutineer
|
|
|
120
157
|
group.each do |i|
|
|
121
158
|
m = mutations[i]
|
|
122
159
|
span = "#{source.byteslice(from...m.start_offset)}#{m.replacement}#{source.byteslice(m.end_offset...to)}"
|
|
123
|
-
keys[i] = [
|
|
160
|
+
keys[i] = [lines[i], Digest::SHA256.digest(span)]
|
|
124
161
|
end
|
|
125
162
|
end
|
|
126
163
|
keys
|
|
@@ -130,10 +167,14 @@ module Mutineer
|
|
|
130
167
|
# those results would treat existing assertion failures as killed mutants.
|
|
131
168
|
#
|
|
132
169
|
# @param coverage_map [Mutineer::CoverageMap] the built or loaded map.
|
|
170
|
+
# @param framework [String, nil] the run's framework (nil is Minitest, as in
|
|
171
|
+
# {TestRunners.for}). For Minitest, the message names the seed: an
|
|
172
|
+
# order-dependent suite can pass in a random order and fail in the
|
|
173
|
+
# pinned one (#282).
|
|
133
174
|
# @return [void]
|
|
134
175
|
# @raise [Mutineer::SmokeCheckError] when any captured test failed clean,
|
|
135
176
|
# or when no test recorded coverage and a capture failed.
|
|
136
|
-
def self.abort_if_unclean!(coverage_map)
|
|
177
|
+
def self.abort_if_unclean!(coverage_map, framework: nil)
|
|
137
178
|
if coverage_map.map.empty? && coverage_map.failed_test_files.any?
|
|
138
179
|
raise SmokeCheckError, "no test recorded coverage, and capture failed for #{coverage_map.failed_test_files.join(', ')}"
|
|
139
180
|
end
|
|
@@ -141,9 +182,47 @@ module Mutineer
|
|
|
141
182
|
files = coverage_map.failed_clean_tests
|
|
142
183
|
return if files.empty?
|
|
143
184
|
|
|
185
|
+
seed_hint = TestRunners.for(framework) == TestRunners::Minitest ? minitest_seed_hint : ""
|
|
144
186
|
raise SmokeCheckError,
|
|
145
187
|
"the unmutated suite is not green (#{files.join(', ')}) — " \
|
|
146
|
-
"#{ExternalBackend.generic_env_hint}
|
|
188
|
+
"#{ExternalBackend.generic_env_hint}.#{seed_hint}"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Names the configured seed, and accounts for helper overrides.
|
|
192
|
+
#
|
|
193
|
+
# @api private
|
|
194
|
+
# @return [String]
|
|
195
|
+
def self.minitest_seed_hint
|
|
196
|
+
seed = MinitestIntegration.seed
|
|
197
|
+
" Mutineer runs Minitest with SEED=#{seed}, unless a test helper changes SEED. " \
|
|
198
|
+
"An order-dependent test can fail here and pass in another order. " \
|
|
199
|
+
"Check the listed test files with the same seed and helpers " \
|
|
200
|
+
"(for \"combined suite\", all of them together). " \
|
|
201
|
+
"Change SEED, including helper settings, to choose another order."
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# The reload mutant text for a def in a concern's `included` or `prepended`
|
|
205
|
+
# block (#297): the mutated file with one statement added just before the
|
|
206
|
+
# hook call, `::Mutineer::ConcernBlocks.store_hook_def do <mutated def> end`.
|
|
207
|
+
# The statement sits in the same body as the hook call, so the stored def has
|
|
208
|
+
# the block's constant scope. After the load, the child passes the hook-def
|
|
209
|
+
# values to {ConcernBlocks.place_hook_def}. Shared by {Runner.run} and
|
|
210
|
+
# {DaemonBackend.job_result}, so both build the same text. The hook call starts
|
|
211
|
+
# before the def, so the mutation does not move its offset. A bad insert is
|
|
212
|
+
# not checked here: the child's `load` raises `SyntaxError` before any test
|
|
213
|
+
# runs, so the mutant scores `error`.
|
|
214
|
+
#
|
|
215
|
+
# @param mutated [String] the mutated source text.
|
|
216
|
+
# @param mutation [Mutineer::Mutation] the mutation.
|
|
217
|
+
# @param subject [Mutineer::Subject] a subject with a {Subject#reload_hook_call}.
|
|
218
|
+
# @param source [String] the original source text.
|
|
219
|
+
# @return [Array(String, Hash{Symbol => Object})] the mutant text, and the
|
|
220
|
+
# original def's `line`, `name` and `singleton`.
|
|
221
|
+
def self.hook_mutant(mutated, mutation, subject, source)
|
|
222
|
+
at = subject.reload_hook_call.location.start_offset
|
|
223
|
+
stored = "::Mutineer::ConcernBlocks.store_hook_def do\n#{Isolation.mutated_def(mutation, subject, source)}\nend; "
|
|
224
|
+
[mutated.byteslice(0...at) + stored + mutated.byteslice(at..),
|
|
225
|
+
{ line: subject.def_node.location.start_line, name: subject.name, singleton: subject.singleton }]
|
|
147
226
|
end
|
|
148
227
|
|
|
149
228
|
# Coverage-based test selection, shared by the in-process ({Runner.run}) and daemon
|
|
@@ -152,11 +231,14 @@ module Mutineer
|
|
|
152
231
|
# order of {CoverageMap#order_tests} (paired files, then cheapest), or
|
|
153
232
|
# `[:verdict, Result]` (no_coverage / uncapturable) when none do. A line
|
|
154
233
|
# Ruby does not count (a later line of a multi-line statement) uses the
|
|
155
|
-
# tests that ran the statement that holds it ({StatementLines}). When
|
|
156
|
-
#
|
|
157
|
-
#
|
|
158
|
-
#
|
|
159
|
-
#
|
|
234
|
+
# tests that ran the statement that holds it ({StatementLines}). When those
|
|
235
|
+
# lines give no tests, it retries with the loose lines of the statement:
|
|
236
|
+
# code of the statement that runs only sometimes (the right side of a
|
|
237
|
+
# multi-line `a && b`, #254), and a first or last line that holds other
|
|
238
|
+
# code (the opener of a one-line block, #289), but never the `def` line.
|
|
239
|
+
# Those tests may not have run the mutant, so the answer is
|
|
240
|
+
# `[:maybe, abs_test_paths]` and {selection_verdict} treats a survivor as
|
|
241
|
+
# no_coverage.
|
|
160
242
|
#
|
|
161
243
|
# An empty selection is `:ran_at_load` when the mutant's line ran while the
|
|
162
244
|
# app booted or its class loaded ({ran_at_load?}): no test ran it, but the
|
|
@@ -182,11 +264,12 @@ module Mutineer
|
|
|
182
264
|
if chosen.empty? && subject
|
|
183
265
|
# A multi-line statement has a count on one of its lines only.
|
|
184
266
|
lines = StatementLines.for(subject.def_node, source, mutation.start_offset)
|
|
185
|
-
|
|
267
|
+
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
268
|
+
if chosen.empty?
|
|
186
269
|
lines = StatementLines.for(subject.def_node, source, mutation.start_offset, loose: true)
|
|
270
|
+
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
187
271
|
kind = :maybe
|
|
188
272
|
end
|
|
189
|
-
chosen = lines.flat_map { |l| coverage_map.tests_for(source_file, l) }.uniq
|
|
190
273
|
end
|
|
191
274
|
if chosen.empty?
|
|
192
275
|
# Method BODY range, not the whole def: the def/end lines are "covered" at
|
|
@@ -435,16 +518,17 @@ module Mutineer
|
|
|
435
518
|
config.sources.map { |f| File.dirname(File.expand_path(f, config.project_root)) }.uniq
|
|
436
519
|
end
|
|
437
520
|
|
|
438
|
-
# Removes stale mutant tempfiles from the given directories. The
|
|
439
|
-
#
|
|
440
|
-
#
|
|
441
|
-
#
|
|
442
|
-
#
|
|
521
|
+
# Removes stale mutant tempfiles from the given directories. The default
|
|
522
|
+
# glob matches the files of both modes, so the in-process boot sweep also
|
|
523
|
+
# removes an old daemon file (#287). {DaemonBackend} passes its daemon-only
|
|
524
|
+
# glob when it sweeps tool-side at the end of every daemon run, empty or not.
|
|
525
|
+
# A file another process still owns is left in place: both modes record the
|
|
526
|
+
# creating pid in the name and lock the file while it loads (#295).
|
|
443
527
|
#
|
|
444
528
|
# @param dirs [Array<String>] directories to sweep.
|
|
445
529
|
# @param glob [String] filename pattern to remove.
|
|
446
530
|
# @return [Array<String>] the files left in place because a process owns them.
|
|
447
|
-
def self.sweep_orphans(dirs, glob =
|
|
531
|
+
def self.sweep_orphans(dirs, glob = OrphanGuard::TEMP_BASENAME)
|
|
448
532
|
dirs.flat_map do |dir|
|
|
449
533
|
Dir.glob(File.join(dir, glob)).select do |path|
|
|
450
534
|
next true if OrphanGuard.mutant_file_in_use?(path)
|
|
@@ -9,6 +9,7 @@ module Mutineer
|
|
|
9
9
|
# ["start"] the recorder is armed; once, first
|
|
10
10
|
# ["pass", file, name, id] a test passed against the mutant
|
|
11
11
|
# ["kill", file, name, id] a test failed or errored: it killed it
|
|
12
|
+
# ["unavailable", file, name, id] a test ran but its database failed
|
|
12
13
|
# ["parallel"] the run reached its parallel tests; once
|
|
13
14
|
# ["skip"] what runs next, a run without --matrix
|
|
14
15
|
# skips (see below)
|
|
@@ -27,20 +28,22 @@ module Mutineer
|
|
|
27
28
|
#
|
|
28
29
|
# Minitest runs its serial test classes first and its `parallelize_me!`
|
|
29
30
|
# classes after them, and the stop at the first failure of a run without
|
|
30
|
-
# `--matrix` skips everything after a serial
|
|
31
|
+
# `--matrix` skips everything after a serial failure. `parallel` marks where the
|
|
31
32
|
# parallel tests begin, so the parent can tell a kill that the stop would have
|
|
32
33
|
# followed from one it could not. `cleanup` marks where the tests end and the
|
|
33
34
|
# suite's own hooks begin (RSpec `after(:suite)`). It decides no verdict
|
|
34
35
|
# (the `skip` regions do); it only marks a later test line as out of order.
|
|
35
36
|
#
|
|
36
|
-
# After the first serial
|
|
37
|
+
# After the first serial failure, a run without `--matrix` skips every later
|
|
37
38
|
# test and every later test class or example group, but it still runs the
|
|
38
39
|
# rest of the code around the failing test: the remainder of its Minitest
|
|
39
40
|
# class `run` wrapper, the `after(:all)` hooks of its RSpec groups, and the
|
|
40
41
|
# suite hooks. The recorder writes `skip` as it enters something that run
|
|
41
42
|
# would skip and `unskip` as that returns, so a child that exits, crashes or
|
|
42
43
|
# hangs inside a skipped region is one the plain run would have stopped
|
|
43
|
-
# before (see {Isolation.finish}).
|
|
44
|
+
# before (see {Isolation.finish}). Its first failure decides the verdict:
|
|
45
|
+
# killed for a real kill, error for an unavailable database outcome. The
|
|
46
|
+
# pairs nest. An unavailable result also leaves the row incomplete.
|
|
44
47
|
#
|
|
45
48
|
# A row is complete only with `start` and `end`, in order, and no lost line: a
|
|
46
49
|
# test that exits the process, a crash, an interrupted run or a recorder that
|
|
@@ -61,6 +64,9 @@ module Mutineer
|
|
|
61
64
|
# Event for a test that failed or errored against the mutant: it killed it.
|
|
62
65
|
KILL = "kill"
|
|
63
66
|
|
|
67
|
+
# Event for a test whose database failed, so it provides no mutation verdict.
|
|
68
|
+
UNAVAILABLE = "unavailable"
|
|
69
|
+
|
|
64
70
|
# Event that opens a recorded run.
|
|
65
71
|
START = "start"
|
|
66
72
|
|
|
@@ -87,17 +93,18 @@ module Mutineer
|
|
|
87
93
|
|
|
88
94
|
# What one child sent. `killed` and `ran` are sorted, unique
|
|
89
95
|
# `[file, name, id]` tests (`ran` includes the killers); `lost` counts lines
|
|
90
|
-
# that could not be read, including a partial last line
|
|
91
|
-
# `cleanup` say the marker arrived
|
|
92
|
-
#
|
|
96
|
+
# that could not be read, including a partial last line, and unavailable
|
|
97
|
+
# outcomes. `parallel` and `cleanup` say the marker arrived. `serial_kill`
|
|
98
|
+
# or `serial_error` says which failure came first before `parallel`.
|
|
99
|
+
# `skipping` counts the {SKIP} regions still
|
|
93
100
|
# open when the stream ended. `invalid` is set by any line out of order.
|
|
94
|
-
Report = Struct.new(:killed, :ran, :started, :finished, :parallel, :cleanup, :serial_kill, :skipping, :invalid,
|
|
101
|
+
Report = Struct.new(:killed, :ran, :started, :finished, :parallel, :cleanup, :serial_kill, :serial_error, :skipping, :invalid,
|
|
95
102
|
:lost, keyword_init: true)
|
|
96
103
|
|
|
97
104
|
# Writes one test line.
|
|
98
105
|
#
|
|
99
106
|
# @param io [IO] the write end of the channel.
|
|
100
|
-
# @param event [String] {PASS} or {
|
|
107
|
+
# @param event [String] {PASS}, {KILL}, or {UNAVAILABLE}.
|
|
101
108
|
# @param file [String] the file that defines the test.
|
|
102
109
|
# @param name [String] the test's display name, e.g. `CalculatorTest#test_add`.
|
|
103
110
|
# @param id [String] what tells the test apart from others with its file and
|
|
@@ -169,7 +176,7 @@ module Mutineer
|
|
|
169
176
|
# @return [Report]
|
|
170
177
|
def self.parse(buffer)
|
|
171
178
|
report = Report.new(killed: [], ran: [], started: false, finished: false, parallel: false, cleanup: false,
|
|
172
|
-
serial_kill: false, skipping: 0, invalid: false, lost: 0)
|
|
179
|
+
serial_kill: false, serial_error: false, skipping: 0, invalid: false, lost: 0)
|
|
173
180
|
buffer.dup.force_encoding(Encoding::UTF_8).each_line do |line|
|
|
174
181
|
fields = line.end_with?("\n") ? parse_line(line) : nil
|
|
175
182
|
fields ? apply(report, fields) : report.lost += 1
|
|
@@ -211,10 +218,13 @@ module Mutineer
|
|
|
211
218
|
report.invalid = true unless open?(report) && !report.cleanup
|
|
212
219
|
test = fields[1, 3]
|
|
213
220
|
report.ran << test
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
221
|
+
if fields[0] == UNAVAILABLE
|
|
222
|
+
report.lost += 1
|
|
223
|
+
report.serial_error = true unless report.parallel || report.serial_kill
|
|
224
|
+
elsif fields[0] == KILL
|
|
225
|
+
report.killed << test
|
|
226
|
+
report.serial_kill = true unless report.parallel || report.serial_error
|
|
227
|
+
end
|
|
218
228
|
end
|
|
219
229
|
end
|
|
220
230
|
|
|
@@ -238,7 +248,7 @@ module Mutineer
|
|
|
238
248
|
|
|
239
249
|
valid =
|
|
240
250
|
case fields[0]
|
|
241
|
-
when PASS, KILL then fields.size == 4
|
|
251
|
+
when PASS, KILL, UNAVAILABLE then fields.size == 4
|
|
242
252
|
when START, PARALLEL, SKIP, UNSKIP, CLEANUP, LOST, FINISH then fields.size == 1
|
|
243
253
|
end
|
|
244
254
|
fields if valid
|
data/lib/mutineer/migrate.rb
CHANGED
|
@@ -7,11 +7,12 @@ require_relative "mutator_registry"
|
|
|
7
7
|
|
|
8
8
|
module Mutineer
|
|
9
9
|
# Rewrites old-format mutant ids in a `.mutineer.yml` `ignore:` list to the
|
|
10
|
-
# current id format. The map is the inverse of the `id_map`
|
|
11
|
-
# {JobPlan.collect_jobs}: each old id goes to every new
|
|
12
|
-
# to it,
|
|
13
|
-
#
|
|
14
|
-
#
|
|
10
|
+
# current id format. The map is the inverse of the `id_map` and
|
|
11
|
+
# `dropped_id_map` from {JobPlan.collect_jobs}: each old id goes to every new
|
|
12
|
+
# id that still hashes to it, reported ids first, each group in collection
|
|
13
|
+
# order. A dropped repeat of an edit is not reported, but its id is still
|
|
14
|
+
# current (#275). The map covers only the configured operators and the
|
|
15
|
+
# sources on the command line. Ids are static hashes, so this runs no tests.
|
|
15
16
|
#
|
|
16
17
|
# The file is edited as text. A YAML load and dump would drop comments and
|
|
17
18
|
# reorder keys. Only a bare id inside `ignore:` is replaced. A flow list may
|
|
@@ -23,7 +24,7 @@ module Mutineer
|
|
|
23
24
|
# one mapping, so it is left in place and reported for a hand edit. A
|
|
24
25
|
# baseline file is never read.
|
|
25
26
|
module Migrate
|
|
26
|
-
# One old id and the new ids that replace it, in
|
|
27
|
+
# One old id and the new ids that replace it, in {Migrate.id_maps} order.
|
|
27
28
|
Replacement = Data.define(:old_id, :new_ids)
|
|
28
29
|
|
|
29
30
|
# The outcome of rewriting one file in memory. `text` is what to write.
|
|
@@ -42,12 +43,13 @@ module Mutineer
|
|
|
42
43
|
def self.id_maps(config)
|
|
43
44
|
names = config.operators || MutatorRegistry::DEFAULT_NAMES
|
|
44
45
|
_jobs, _ignored, _source_map, extras = JobPlan.collect_jobs(config, MutatorRegistry.resolve(names))
|
|
46
|
+
all = extras[:id_map].merge(extras[:dropped_id_map])
|
|
45
47
|
old_to_new = {}
|
|
46
|
-
|
|
48
|
+
all.each do |new_id, old_id|
|
|
47
49
|
list = (old_to_new[old_id] ||= [])
|
|
48
50
|
list << new_id unless list.include?(new_id)
|
|
49
51
|
end
|
|
50
|
-
[
|
|
52
|
+
[all.keys.to_set, old_to_new]
|
|
51
53
|
end
|
|
52
54
|
|
|
53
55
|
# Rewrites `ignore:` ids in `text`. An id that is already a current new id
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "outer_reporter"
|
|
4
|
+
require_relative "../redaction"
|
|
5
|
+
|
|
6
|
+
module Mutineer
|
|
7
|
+
class MinitestIntegration
|
|
8
|
+
# Scores a run as `error`, not `killed`, when its first failing test failed
|
|
9
|
+
# because of an Active Record connection or lock error (#296). Such a
|
|
10
|
+
# failure says the database broke, not that a test caught the mutant.
|
|
11
|
+
# Child-process only, like MinitestIntegration.
|
|
12
|
+
#
|
|
13
|
+
# Only the first failed result on the outer reporter counts (see
|
|
14
|
+
# {OuterReporter}). A run that stops at its first failure ends there, so a
|
|
15
|
+
# full run and a `--matrix` run give the same verdict. One limit: the tests
|
|
16
|
+
# of a `parallelize_me!` class record in the order their threads finish, so
|
|
17
|
+
# a mutant that makes one parallel test hit a database error and another
|
|
18
|
+
# fail an assertion can score `error` in one run and `killed` in the next.
|
|
19
|
+
# Stdlib only: the app-side daemon loads this file without Prism. An
|
|
20
|
+
# unknown Minitest shape arms nothing, and the run keeps its 0 or 1.
|
|
21
|
+
module InTestDatabaseError
|
|
22
|
+
extend OuterReporter::Client
|
|
23
|
+
|
|
24
|
+
# The Active Record errors that mean the database, not the test, failed.
|
|
25
|
+
# Subclasses count too: `ConnectionTimeoutError` (a pool timeout) and
|
|
26
|
+
# `DatabaseConnectionError` are `ConnectionNotEstablished`. Looked up by
|
|
27
|
+
# name when a result is recorded, so a project without Active Record, or
|
|
28
|
+
# a Rails without `ConnectionFailed` (before 7.1), matches nothing. Other
|
|
29
|
+
# `StatementInvalid` errors are not here: a mutant can cause bad SQL.
|
|
30
|
+
NAMES = %w[
|
|
31
|
+
ActiveRecord::ConnectionNotEstablished
|
|
32
|
+
ActiveRecord::ConnectionFailed
|
|
33
|
+
ActiveRecord::Deadlocked
|
|
34
|
+
ActiveRecord::LockWaitTimeout
|
|
35
|
+
].freeze
|
|
36
|
+
|
|
37
|
+
# Subclasses of {NAMES} that point at the code, not the database. Rails 8
|
|
38
|
+
# raises `ConnectionNotDefined` for a role or shard that no connection
|
|
39
|
+
# defines, which a mutant of a `connected_to` call can cause. Rails 7.2
|
|
40
|
+
# raises a plain `ConnectionNotEstablished` there, so on 7.2 such a mutant
|
|
41
|
+
# scores `error`. The message is not matched (plan Q2).
|
|
42
|
+
CODE_ERROR_NAMES = %w[ActiveRecord::ConnectionNotDefined].freeze
|
|
43
|
+
|
|
44
|
+
# The lock errors in {NAMES}, which get their own stderr hint.
|
|
45
|
+
LOCK_ERROR_NAMES = %w[ActiveRecord::Deadlocked ActiveRecord::LockWaitTimeout].freeze
|
|
46
|
+
|
|
47
|
+
# Prepended on `Minitest::CompositeReporter`.
|
|
48
|
+
module RecordFirstFailure
|
|
49
|
+
# Records the result, then lets the module keep the first failure.
|
|
50
|
+
#
|
|
51
|
+
# @param result [Minitest::Result] the result of one test.
|
|
52
|
+
# @return [void]
|
|
53
|
+
def record(result)
|
|
54
|
+
super
|
|
55
|
+
InTestDatabaseError.record(self, result)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
class << self
|
|
60
|
+
# The first failed result's database error, false when that result
|
|
61
|
+
# failed for another reason, or nil before any failure.
|
|
62
|
+
#
|
|
63
|
+
# @return [Exception, false, nil]
|
|
64
|
+
attr_accessor :first_failure
|
|
65
|
+
|
|
66
|
+
# Installs the hooks and arms the module for this process.
|
|
67
|
+
#
|
|
68
|
+
# @return [Boolean] false when the Minitest shape is unknown.
|
|
69
|
+
def arm!
|
|
70
|
+
outer = OuterReporter.hook_for_loaded_minitest
|
|
71
|
+
return false unless outer
|
|
72
|
+
|
|
73
|
+
OuterReporter.prepend_once(::Minitest::CompositeReporter, RecordFirstFailure)
|
|
74
|
+
OuterReporter.prepend_once(::Minitest.singleton_class, outer)
|
|
75
|
+
OuterReporter.register(self)
|
|
76
|
+
self.armed_pid = Process.pid
|
|
77
|
+
self.armed_reporter = nil
|
|
78
|
+
self.first_failure = nil
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Clears the armed state.
|
|
83
|
+
#
|
|
84
|
+
# @return [void]
|
|
85
|
+
def disarm!
|
|
86
|
+
self.armed_pid = nil
|
|
87
|
+
self.armed_reporter = nil
|
|
88
|
+
self.first_failure = nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Keeps the first failed result recorded on the outer reporter. Minitest
|
|
92
|
+
# records from several threads only under its own reporter lock. A
|
|
93
|
+
# result it cannot read counts as an ordinary failure, so a fault here
|
|
94
|
+
# keeps the old verdict and never turns a kill into an error.
|
|
95
|
+
#
|
|
96
|
+
# @param reporter [Minitest::CompositeReporter] the recording reporter.
|
|
97
|
+
# @param result [Minitest::Result] the result of one test.
|
|
98
|
+
# @return [void]
|
|
99
|
+
def record(reporter, result)
|
|
100
|
+
return if result.passed? || result.skipped?
|
|
101
|
+
return unless first_failure.nil? && outer_reporter?(reporter)
|
|
102
|
+
|
|
103
|
+
self.first_failure = database_error(result.failures.first) || false
|
|
104
|
+
rescue StandardError
|
|
105
|
+
self.first_failure = false
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The run's exit code: 2 instead of 1 when the armed run's first failure
|
|
109
|
+
# was a database error. Then it also writes one line on stderr that
|
|
110
|
+
# names the error. `STDERR`, not `warn`: a test can leave `$stderr` as
|
|
111
|
+
# a `StringIO`. The flush matters because both fork boundaries end with
|
|
112
|
+
# `exit!`.
|
|
113
|
+
#
|
|
114
|
+
# @param code [Integer] 0 or 1 from the run.
|
|
115
|
+
# @return [Integer] `code`, or 2.
|
|
116
|
+
def exit_code(code)
|
|
117
|
+
error = first_failure if armed_here?
|
|
118
|
+
return code unless code == 1 && error
|
|
119
|
+
|
|
120
|
+
STDERR.puts(line(error))
|
|
121
|
+
STDERR.flush
|
|
122
|
+
2
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# The database error a failure wraps. The first entry of a result is
|
|
126
|
+
# the error that ended the test: Minitest records the setup or body
|
|
127
|
+
# error before a teardown error. The cause chain is not read: Active
|
|
128
|
+
# Record already translates driver errors into the listed classes.
|
|
129
|
+
#
|
|
130
|
+
# Minitest before 5.14 names the wrapped error `exception`.
|
|
131
|
+
#
|
|
132
|
+
# Shared with the matrix recorder so an unavailable database outcome
|
|
133
|
+
# cannot be named as a mutation kill.
|
|
134
|
+
#
|
|
135
|
+
# @api private
|
|
136
|
+
# @param failure [Minitest::Assertion, nil] the result's first entry.
|
|
137
|
+
# @return [Exception, nil]
|
|
138
|
+
def database_error(failure)
|
|
139
|
+
return unless failure.is_a?(::Minitest::UnexpectedError)
|
|
140
|
+
|
|
141
|
+
error = failure.respond_to?(:error) ? failure.error : failure.exception
|
|
142
|
+
return if loaded(CODE_ERROR_NAMES).any? { |klass| error.is_a?(klass) }
|
|
143
|
+
|
|
144
|
+
error if loaded(NAMES).any? { |klass| error.is_a?(klass) }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
private
|
|
148
|
+
|
|
149
|
+
# The named classes this process has loaded.
|
|
150
|
+
#
|
|
151
|
+
# @param names [Array<String>] constant names.
|
|
152
|
+
# @return [Array<Class>]
|
|
153
|
+
def loaded(names)
|
|
154
|
+
names.filter_map { |name| Object.const_get(name) if Object.const_defined?(name) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The stderr line. Redacted before the cut to one line, so a cut can
|
|
158
|
+
# never split a password away from its URL. The wording fits every run,
|
|
159
|
+
# the clean check included, and a mutant can cause the error too.
|
|
160
|
+
#
|
|
161
|
+
# @param error [Exception] the database error.
|
|
162
|
+
# @return [String]
|
|
163
|
+
def line(error)
|
|
164
|
+
text = Redaction.text("#{error.class}: #{error.message}".scrub)
|
|
165
|
+
text = text.lines.first.to_s.chomp.delete_suffix(".")
|
|
166
|
+
"[mutineer-child] a test failed with a database error, so the run has no verdict: #{text}. #{hint(error)}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# What to check for the error: a lock error comes from a second
|
|
170
|
+
# connection or thread, any other from the server.
|
|
171
|
+
#
|
|
172
|
+
# @param error [Exception] the database error.
|
|
173
|
+
# @return [String]
|
|
174
|
+
def hint(error)
|
|
175
|
+
if loaded(LOCK_ERROR_NAMES).any? { |klass| error.is_a?(klass) }
|
|
176
|
+
"Look for a test or app code that holds a lock on a second connection or thread."
|
|
177
|
+
else
|
|
178
|
+
"Check that the database server is up and has free connections."
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|