mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
|
@@ -6,6 +6,7 @@ require_relative "coverage_map"
|
|
|
6
6
|
require_relative "daemon_client"
|
|
7
7
|
require_relative "progress"
|
|
8
8
|
require_relative "job_plan"
|
|
9
|
+
require_relative "orphan_guard"
|
|
9
10
|
|
|
10
11
|
module Mutineer
|
|
11
12
|
# Daemon execution backend. Boots the app ONCE in a persistent subprocess under
|
|
@@ -31,8 +32,16 @@ module Mutineer
|
|
|
31
32
|
DEFAULT_TIMEOUT = 60
|
|
32
33
|
|
|
33
34
|
# The daemon's per-mutant tempfile, written into the source dir so
|
|
34
|
-
# require_relative resolves.
|
|
35
|
-
|
|
35
|
+
# require_relative resolves. The leading dot hides it from Zeitwerk (#253);
|
|
36
|
+
# older versions wrote it without one. Daemon files only, unlike the boot
|
|
37
|
+
# sweeps ({OrphanGuard::TEMP_BASENAME}): this sweep runs after the boot and
|
|
38
|
+
# waits up to {SWEEP_GRACE} while any file it matches is still owned, so a
|
|
39
|
+
# parallel in-process run's live files would add that wait to every daemon run.
|
|
40
|
+
DAEMON_TEMP_GLOB = "{.,}mutineer_daemon#{"[0-9]" * 8}-*.rb"
|
|
41
|
+
|
|
42
|
+
# Seconds the end-of-run sweep waits for an orphaned child to die. Its
|
|
43
|
+
# watchdog notices a dead daemon within {OrphanGuard::POLL}.
|
|
44
|
+
SWEEP_GRACE = (OrphanGuard::POLL * 2) + 0.5
|
|
36
45
|
|
|
37
46
|
# Full daemon run: collect jobs, build the coverage map once, then execute
|
|
38
47
|
# serially or across N worker daemons. Fail-fast forces serial so the survivor
|
|
@@ -44,25 +53,14 @@ module Mutineer
|
|
|
44
53
|
# source map, and the {JobPlan.collect_jobs} extras.
|
|
45
54
|
def self.execute(config, operator_classes)
|
|
46
55
|
jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
|
|
47
|
-
jobs = JobPlan.
|
|
56
|
+
jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
|
|
48
57
|
abs_tests = config.tests.map { |t| File.expand_path(t, config.project_root) }
|
|
49
58
|
|
|
50
59
|
# Nothing to mutate (`--since` matched no changed line, or every mutant is
|
|
51
60
|
# suppressed). Return before booting anything: the coverage daemon and the
|
|
52
61
|
# worker daemons below each boot the whole app, and README documents
|
|
53
62
|
# `--since origin/<base>` for PR CI, where a docs-only PR is routine.
|
|
54
|
-
if jobs.empty?
|
|
55
|
-
# The daemon sweeps orphaned temps at boot and nothing boots here, so sweep
|
|
56
|
-
# tool-side. A file a hard-killed run left in app/models breaks the app's own
|
|
57
|
-
# Zeitwerk boot, not just Mutineer's next run.
|
|
58
|
-
JobPlan.sweep_orphans(JobPlan.source_dirs(config), DAEMON_TEMP_GLOB)
|
|
59
|
-
return [AggregateResult.new(ignored_results), source_map, extras]
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
# Build the coverage map once (app-side). nil when the build fails: runners
|
|
63
|
-
# fall back to the full --test set (and emit a stderr warning) rather than
|
|
64
|
-
# mis-scoring everything as no_coverage.
|
|
65
|
-
coverage_map = build_coverage_map(config, abs_tests)
|
|
63
|
+
return [AggregateResult.new(ignored_results), source_map, extras] if jobs.empty?
|
|
66
64
|
|
|
67
65
|
# Worker count = resolved --jobs, capped at the job count (no idle daemons).
|
|
68
66
|
# >1 → N concurrent daemon handles, each on its OWN worker DB (N-handles, the
|
|
@@ -71,18 +69,60 @@ module Mutineer
|
|
|
71
69
|
# input index, so the verdict set would diverge from serial (a different,
|
|
72
70
|
# non-deterministic survivor set/score). The "identical to --jobs 1" guarantee
|
|
73
71
|
# below only holds when fail-fast cannot race.
|
|
72
|
+
# A second database config is only known after the app boots, so the first
|
|
73
|
+
# daemon can still lower this count before any worker starts.
|
|
74
74
|
worker_count = [config.jobs || 1, 1].max
|
|
75
75
|
worker_count = 1 if config.fail_fast
|
|
76
76
|
worker_count = [worker_count, jobs.size].min
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
78
|
+
# One booted process copies worker databases, then builds the coverage map.
|
|
79
|
+
# A cached map still provisions: the copy has to happen while no worker is
|
|
80
|
+
# connected to the template database. That process keeps its database lock
|
|
81
|
+
# until the worker daemons have taken theirs.
|
|
82
|
+
prepare_client = nil
|
|
83
|
+
begin
|
|
84
|
+
coverage_map, worker_count, prepare_client = prepare_run(config, abs_tests, worker_count)
|
|
85
|
+
|
|
86
|
+
results =
|
|
87
|
+
if worker_count > 1
|
|
88
|
+
run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client)
|
|
89
|
+
else
|
|
90
|
+
run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client)
|
|
91
|
+
end
|
|
92
|
+
ensure
|
|
93
|
+
# run_serial and run_parallel quit this client once workers hold locks.
|
|
94
|
+
# Quit again here when they never reached that point. A second quit is a no-op.
|
|
95
|
+
prepare_client&.quit
|
|
96
|
+
end
|
|
84
97
|
|
|
85
98
|
[AggregateResult.new(results + ignored_results), source_map, extras]
|
|
99
|
+
ensure
|
|
100
|
+
sweep_daemon_temps(config)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Removes daemon mutant files left in the source dirs, on every exit path.
|
|
104
|
+
# The daemon sweeps at boot, but an empty run boots nothing, and a daemon
|
|
105
|
+
# that dies mid-mutant orphans a child that lives on until its watchdog
|
|
106
|
+
# fires (#250). The respawned daemon sweeps before that, and a run that
|
|
107
|
+
# ends or aborts right after the crash does too. So a file whose owner is
|
|
108
|
+
# still alive gets {SWEEP_GRACE} to go. A clean run leaves no file and pays
|
|
109
|
+
# nothing. A file a concurrent run still owns stays in place, but looks the
|
|
110
|
+
# same as a dying child's, so this run then waits the full grace.
|
|
111
|
+
#
|
|
112
|
+
# @param config [Mutineer::Config] the run config.
|
|
113
|
+
# @return [void]
|
|
114
|
+
def self.sweep_daemon_temps(config)
|
|
115
|
+
dirs = JobPlan.source_dirs(config)
|
|
116
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + SWEEP_GRACE
|
|
117
|
+
loop do
|
|
118
|
+
still_owned = JobPlan.sweep_orphans(dirs, DAEMON_TEMP_GLOB)
|
|
119
|
+
break if still_owned.empty?
|
|
120
|
+
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
121
|
+
|
|
122
|
+
sleep 0.1
|
|
123
|
+
end
|
|
124
|
+
rescue SystemCallError
|
|
125
|
+
nil # never replace the run's own result or exception with a cleanup error
|
|
86
126
|
end
|
|
87
127
|
|
|
88
128
|
# Build the coverage map via a short-lived daemon (boots the app once, captures
|
|
@@ -101,11 +141,109 @@ module Mutineer
|
|
|
101
141
|
def self.build_coverage_map(config, abs_tests)
|
|
102
142
|
client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
|
|
103
143
|
app_root: config.project_root).start
|
|
104
|
-
|
|
105
|
-
client
|
|
144
|
+
begin
|
|
145
|
+
coverage_map_from_client(client, config)
|
|
106
146
|
ensure
|
|
107
147
|
client.quit
|
|
108
148
|
end
|
|
149
|
+
rescue DaemonBootTimeout
|
|
150
|
+
raise
|
|
151
|
+
rescue DaemonBootError => e
|
|
152
|
+
warn_coverage_fallback("#{e.class}: #{e.message}")
|
|
153
|
+
nil
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Boot the first daemon, copy worker databases, then read the coverage map
|
|
157
|
+
# from that same process. A boot failure still falls back to the full test
|
|
158
|
+
# set. A provisioning failure does not: it stops the run.
|
|
159
|
+
#
|
|
160
|
+
# @param config [Mutineer::Config]
|
|
161
|
+
# @param abs_tests [Array<String>]
|
|
162
|
+
# @param worker_count [Integer] slots requested before the database count is known.
|
|
163
|
+
# @return [Array] the coverage map (or nil), the worker count to start, and
|
|
164
|
+
# the prepare daemon. The caller quits that daemon after worker daemons
|
|
165
|
+
# hold their locks. It is nil when this process could not boot one.
|
|
166
|
+
def self.prepare_run(config, abs_tests, worker_count)
|
|
167
|
+
client = nil
|
|
168
|
+
begin
|
|
169
|
+
client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
|
|
170
|
+
app_root: config.project_root).start
|
|
171
|
+
rescue DaemonBootTimeout
|
|
172
|
+
raise
|
|
173
|
+
rescue DaemonBootError => e
|
|
174
|
+
warn_coverage_fallback("#{e.class}: #{e.message}")
|
|
175
|
+
# No daemon listed a Rails app's databases, so no second worker may
|
|
176
|
+
# share them. A --boot app without --rails has no database routing.
|
|
177
|
+
unknown = { "error" => "the first daemon did not boot" }
|
|
178
|
+
return [nil, config.rails ? worker_count_for(unknown, [], worker_count) : worker_count, nil]
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
begin
|
|
182
|
+
# Raises ConfigError when shared_databases lists the primary database.
|
|
183
|
+
worker_count = worker_count_for(client.databases, config.shared_databases, worker_count)
|
|
184
|
+
# Raises DaemonBootError. Callers must not score mutants after this.
|
|
185
|
+
client.provision(worker_count)
|
|
186
|
+
[coverage_map_from_client(client, config), worker_count, client]
|
|
187
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
188
|
+
# An interrupt must also quit it, or the daemon keeps its database lock.
|
|
189
|
+
client.quit
|
|
190
|
+
raise
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# The worker count once the booted app's databases are known. Each worker
|
|
195
|
+
# copies only the primary database, so a second database forces one worker
|
|
196
|
+
# unless `shared_databases` lists it. Listing the primary is a config error.
|
|
197
|
+
# A failed lookup also forces one worker: a database nobody counted must
|
|
198
|
+
# never be shared by accident.
|
|
199
|
+
#
|
|
200
|
+
# @param databases [Hash, nil] the daemon's report: "names" and "primary",
|
|
201
|
+
# or "error" (with "primary" when it resolves). nil when the app has no
|
|
202
|
+
# worker-DB adapter.
|
|
203
|
+
# @param shared [Array<String>] the `shared_databases` config names.
|
|
204
|
+
# @param worker_count [Integer] the workers requested.
|
|
205
|
+
# @return [Integer] the workers to start.
|
|
206
|
+
# @raise [Mutineer::ConfigError] when `shared` lists the primary database.
|
|
207
|
+
def self.worker_count_for(databases, shared, worker_count)
|
|
208
|
+
return worker_count if databases.nil?
|
|
209
|
+
|
|
210
|
+
primary = databases["primary"]
|
|
211
|
+
if shared.include?(primary)
|
|
212
|
+
raise ConfigError, "shared_databases lists #{primary}, the database each worker copies. " \
|
|
213
|
+
"List only the other databases, the ones parallel workers may share."
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
if databases["error"]
|
|
217
|
+
if worker_count > 1
|
|
218
|
+
warn "[mutineer] could not list this app's databases (#{databases["error"]}); Mutineer runs one worker."
|
|
219
|
+
end
|
|
220
|
+
return 1
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
names = Array(databases["names"])
|
|
224
|
+
unknown = shared - names
|
|
225
|
+
if unknown.any?
|
|
226
|
+
warn "[mutineer] shared_databases names no test database: #{unknown.join(", ")}. " \
|
|
227
|
+
"This app has #{names.join(", ")}."
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
unshared = names - shared
|
|
231
|
+
return worker_count unless unshared.size > 1 && worker_count > 1
|
|
232
|
+
|
|
233
|
+
outside = shared.empty? ? "" : " outside shared_databases"
|
|
234
|
+
warn "[mutineer] this app has #{unshared.size} databases#{outside}; Mutineer runs one worker. " \
|
|
235
|
+
"Parallel runs support one database. To run in parallel, list each one " \
|
|
236
|
+
"your tests never use under shared_databases: #{(unshared - [primary]).join(", ")}."
|
|
237
|
+
1
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Turn one daemon's coverage reply into a map. Does not start or stop the daemon.
|
|
241
|
+
#
|
|
242
|
+
# @param client [Mutineer::DaemonClient]
|
|
243
|
+
# @param config [Mutineer::Config]
|
|
244
|
+
# @return [Mutineer::CoverageMap, nil]
|
|
245
|
+
def self.coverage_map_from_client(client, config)
|
|
246
|
+
data = client.coverage
|
|
109
247
|
# A red unmutated suite must abort, even when the shipped map is empty.
|
|
110
248
|
# Falling back to the full --test set would treat those failures as kills.
|
|
111
249
|
if data.is_a?(Hash) && Array(data["failed_clean_tests"]).any?
|
|
@@ -114,7 +252,7 @@ module Mutineer
|
|
|
114
252
|
failed_test_files: data["failed_test_files"] || [],
|
|
115
253
|
project_root: config.project_root,
|
|
116
254
|
failed_clean_tests: data["failed_clean_tests"]
|
|
117
|
-
))
|
|
255
|
+
), framework: config.framework)
|
|
118
256
|
end
|
|
119
257
|
|
|
120
258
|
unless data && !(data["map"] || {}).empty?
|
|
@@ -155,59 +293,70 @@ module Mutineer
|
|
|
155
293
|
# (stop at the first survivor).
|
|
156
294
|
#
|
|
157
295
|
# @api private
|
|
296
|
+
# @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
|
|
297
|
+
# the databases. Quit once this worker has booted and taken its lock.
|
|
158
298
|
# @return [Array<Mutineer::Result>] results in input order.
|
|
159
|
-
def self.run_serial(jobs, config, abs_tests, coverage_map, source_map)
|
|
160
|
-
client =
|
|
161
|
-
app_root: config.project_root).start
|
|
162
|
-
results = []
|
|
163
|
-
progress = Progress.new(jobs.size)
|
|
299
|
+
def self.run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client = nil)
|
|
300
|
+
client = nil
|
|
164
301
|
begin
|
|
302
|
+
client = DaemonClient.new(boot: boot_config(config, abs_tests),
|
|
303
|
+
app_root: config.project_root).start
|
|
304
|
+
prepare_client&.quit
|
|
305
|
+
prepare_client = nil
|
|
306
|
+
results = []
|
|
307
|
+
progress = Progress.new(jobs.size)
|
|
165
308
|
jobs.each_with_index do |job, i|
|
|
166
309
|
r = job_result(job, i, client, 0, config, coverage_map, abs_tests, source_map)
|
|
167
310
|
results << r
|
|
168
311
|
progress.tick
|
|
169
312
|
break if config.fail_fast && r.survived?
|
|
170
313
|
end
|
|
314
|
+
results
|
|
171
315
|
ensure
|
|
172
|
-
|
|
316
|
+
prepare_client&.quit
|
|
317
|
+
client&.quit
|
|
173
318
|
end
|
|
174
|
-
results
|
|
175
319
|
end
|
|
176
320
|
|
|
177
321
|
# Parallel path: N daemon handles, each pinned to its own worker slot (own DB).
|
|
178
322
|
# A shared queue of job indices feeds N tool-side threads; results are placed by
|
|
179
323
|
# input index so the verdict set matches serial. Callers must not pass fail_fast
|
|
180
324
|
# here ({execute} forces serial for fail-fast). Per-mutant crashes are classified
|
|
181
|
-
# in {job_result}, shared with the serial path
|
|
182
|
-
#
|
|
325
|
+
# in {job_result}, shared with the serial path. Any error that escapes a worker
|
|
326
|
+
# (a {DaemonBootError}, or a defect) clears the queue, and the run ends once
|
|
327
|
+
# every worker has joined, rather than scoring the remainder first.
|
|
183
328
|
#
|
|
184
329
|
# @api private
|
|
330
|
+
# @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
|
|
331
|
+
# the databases. Quit once every worker has booted and taken its lock.
|
|
185
332
|
# @return [Array<Mutineer::Result>] one result per input job, in input order.
|
|
186
|
-
def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
|
|
333
|
+
def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client = nil)
|
|
187
334
|
results = Array.new(jobs.size)
|
|
188
335
|
progress = Progress.new(jobs.size)
|
|
189
336
|
queue = Queue.new
|
|
337
|
+
errors = Queue.new
|
|
190
338
|
jobs.each_index { |i| queue << i }
|
|
191
339
|
|
|
192
|
-
#
|
|
193
|
-
#
|
|
340
|
+
# Both lists grow one entry at a time, so a refused spawn part-way (EMFILE
|
|
341
|
+
# under a high --jobs) or a Ctrl-C leaves the ensure below every daemon
|
|
342
|
+
# already up. Each client joins its list before it starts, so one
|
|
343
|
+
# interrupted mid-boot is quit too. The prepare daemon stays up through
|
|
344
|
+
# this loop so its lock still covers the databases until each worker
|
|
345
|
+
# holds one.
|
|
194
346
|
clients = []
|
|
347
|
+
threads = []
|
|
195
348
|
begin
|
|
196
349
|
worker_count.times do
|
|
197
|
-
clients << DaemonClient.new(boot: boot_config(config, abs_tests),
|
|
198
|
-
app_root: config.project_root)
|
|
350
|
+
clients << DaemonClient.new(boot: boot_config(config, abs_tests, parallel: true),
|
|
351
|
+
app_root: config.project_root)
|
|
352
|
+
clients.last.start
|
|
199
353
|
end
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
raise
|
|
354
|
+
ensure
|
|
355
|
+
prepare_client&.quit
|
|
203
356
|
end
|
|
204
357
|
|
|
205
|
-
clients.each_with_index
|
|
206
|
-
Thread.new do
|
|
207
|
-
# The abort below is re-raised by join and reported once there; without
|
|
208
|
-
# this Ruby also dumps the thread's backtrace, which the serial path never
|
|
209
|
-
# does. Same fault, same output, whatever --jobs is set to.
|
|
210
|
-
Thread.current.report_on_exception = false
|
|
358
|
+
clients.each_with_index do |client, worker|
|
|
359
|
+
threads << Thread.new do
|
|
211
360
|
loop do
|
|
212
361
|
i = begin
|
|
213
362
|
queue.pop(true)
|
|
@@ -217,20 +366,35 @@ module Mutineer
|
|
|
217
366
|
results[i] = job_result(jobs[i], i, client, worker, config, coverage_map, abs_tests, source_map)
|
|
218
367
|
progress.tick
|
|
219
368
|
end
|
|
220
|
-
rescue
|
|
221
|
-
# The daemon gave up for good. Stop feeding the
|
|
222
|
-
#
|
|
223
|
-
#
|
|
369
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
370
|
+
# The daemon gave up for good, or a defect escaped. Stop feeding the
|
|
371
|
+
# other workers, so they stop after their current mutant, and hand the
|
|
372
|
+
# error to the main thread. The thread itself ends cleanly.
|
|
224
373
|
queue.clear
|
|
225
|
-
|
|
374
|
+
errors << e
|
|
226
375
|
ensure
|
|
227
376
|
client.quit
|
|
228
377
|
end
|
|
229
|
-
end
|
|
378
|
+
end
|
|
379
|
+
# Join every worker before raising, so all daemons have quit and the
|
|
380
|
+
# end-of-run sweep sees each child gone.
|
|
381
|
+
threads.each(&:join)
|
|
382
|
+
raise errors.pop unless errors.empty?
|
|
230
383
|
|
|
231
384
|
# Every job was popped by some worker and every pop assigns, so no slot can
|
|
232
|
-
# be nil here:
|
|
385
|
+
# be nil here: a worker error is raised above instead.
|
|
233
386
|
results
|
|
387
|
+
ensure
|
|
388
|
+
# An exception from outside (Ctrl-C, a caller's Timeout) skips the joins
|
|
389
|
+
# above. Kill every worker first, so their daemons quit at the same time,
|
|
390
|
+
# then wait for them. A second Ctrl-C during that wait still exits.
|
|
391
|
+
queue&.clear
|
|
392
|
+
threads&.each(&:kill)
|
|
393
|
+
threads&.each(&:join)
|
|
394
|
+
# Quit every client too. One may have no thread yet (startup failed or
|
|
395
|
+
# was interrupted), and a thread killed before its block began never runs
|
|
396
|
+
# its own ensure. A second quit of a client is a no-op.
|
|
397
|
+
clients&.each(&:quit)
|
|
234
398
|
end
|
|
235
399
|
|
|
236
400
|
# Build the payload for one job, run it on the given daemon/worker, and attach
|
|
@@ -265,14 +429,24 @@ module Mutineer
|
|
|
265
429
|
Result.skipped
|
|
266
430
|
elsif sel && sel[0] == :verdict
|
|
267
431
|
sel[1]
|
|
432
|
+
elsif subject.unplaceable?("reload") # the daemon always reloads the whole file
|
|
433
|
+
# A :maybe mutant that cannot be placed stays no_coverage, as in-process.
|
|
434
|
+
sel && sel[0] == :maybe ? Result.no_coverage : Result.unplaceable
|
|
268
435
|
else
|
|
436
|
+
payload = { "code" => mutated, "source_file" => File.expand_path(subject.file, config.project_root) }
|
|
437
|
+
if subject.reload_hook_call
|
|
438
|
+
# #297: the daemon child places the hook-block def, as in-process (Runner.run).
|
|
439
|
+
payload["code"], hook_def = JobPlan.hook_mutant(mutated, mutation, subject, source)
|
|
440
|
+
payload["hook_def"] = hook_def.to_h { |key, value| [key.to_s, key == :name ? value.to_s : value] }
|
|
441
|
+
end
|
|
269
442
|
verdict = client.request(
|
|
270
443
|
id: req_id, worker: worker, timeout: config.daemon_timeout || DEFAULT_TIMEOUT,
|
|
271
|
-
payload:
|
|
272
|
-
tests: sel ? sel[1] : abs_tests
|
|
444
|
+
payload: payload, tests: sel ? sel[1] : abs_tests
|
|
273
445
|
)
|
|
274
446
|
# A survivor whose line ran at load, as in-process (Runner.run).
|
|
275
|
-
JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
|
|
447
|
+
loaded = JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
|
|
448
|
+
# Tests that may not have run the mutant, as in-process (#254).
|
|
449
|
+
JobPlan.selection_verdict(sel && sel[0], loaded)
|
|
276
450
|
end
|
|
277
451
|
r.with(subject: subject, mutation: mutation, id: id)
|
|
278
452
|
end
|
|
@@ -285,8 +459,12 @@ module Mutineer
|
|
|
285
459
|
# @param config [Mutineer::Config] the run config.
|
|
286
460
|
# @param abs_tests [Array<String>] absolute --test paths.
|
|
287
461
|
# @param coverage [Boolean] whether this daemon builds the coverage map.
|
|
462
|
+
# @param db_role [String, nil] "prepare" (copy databases on command),
|
|
463
|
+
# "worker" (lock only), or nil to pick from `coverage`.
|
|
464
|
+
# @param parallel [Boolean] whether this daemon is one of several workers.
|
|
465
|
+
# Only then does it check `shared_databases` after each mutant.
|
|
288
466
|
# @return [Hash] the boot config shipped to the daemon.
|
|
289
|
-
def self.boot_config(config, abs_tests, coverage: false)
|
|
467
|
+
def self.boot_config(config, abs_tests, coverage: false, db_role: nil, parallel: false)
|
|
290
468
|
{
|
|
291
469
|
project_root: config.project_root,
|
|
292
470
|
boot: File.expand_path(config.boot || "config/environment", config.project_root),
|
|
@@ -298,14 +476,17 @@ module Mutineer
|
|
|
298
476
|
source_dirs: JobPlan.source_dirs(config), # so the daemon can sweep orphan mutant temps
|
|
299
477
|
framework: config.framework,
|
|
300
478
|
rails: config.rails,
|
|
301
|
-
#
|
|
302
|
-
#
|
|
303
|
-
|
|
479
|
+
# Databases the workers share. A parallel worker fails a mutant whose
|
|
480
|
+
# tests used one. One worker shares nothing, so it gets no names.
|
|
481
|
+
shared_databases: parallel ? config.shared_databases : [],
|
|
304
482
|
# Coverage narrowing. Only the short-lived map-building daemon starts
|
|
305
483
|
# Coverage (before boot); worker daemons boot with it OFF (no wasted
|
|
306
484
|
# instrumentation/memory across every mutant fork). `sources`/`tests` are the
|
|
307
485
|
# map-build inputs.
|
|
308
486
|
coverage: coverage,
|
|
487
|
+
# "prepare" copies worker databases when asked. "worker" only locks.
|
|
488
|
+
# A direct client that omits this provisions at boot instead.
|
|
489
|
+
db_role: db_role || (coverage ? "prepare" : "worker"),
|
|
309
490
|
# The map-building daemon's capture limit (--capture-timeout); nil = default.
|
|
310
491
|
capture_timeout: config.capture_timeout,
|
|
311
492
|
sources: config.sources.map { |s| File.expand_path(s, config.project_root) },
|
|
@@ -313,19 +494,6 @@ module Mutineer
|
|
|
313
494
|
}
|
|
314
495
|
end
|
|
315
496
|
|
|
316
|
-
# Absolute path to the app's `db/schema.rb` if it exists, else nil. Each worker
|
|
317
|
-
# DB starts as a copy of the test DB; the daemon loads this file over the copy
|
|
318
|
-
# when the copy's schema differs. `structure.sql` apps get nil and keep the
|
|
319
|
-
# copy as it is.
|
|
320
|
-
#
|
|
321
|
-
# @param config [Mutineer::Config] the run config.
|
|
322
|
-
# @api private
|
|
323
|
-
# @return [String, nil] absolute schema path or nil.
|
|
324
|
-
def self.schema_path(config)
|
|
325
|
-
path = File.expand_path("db/schema.rb", config.project_root)
|
|
326
|
-
File.exist?(path) ? path : nil
|
|
327
|
-
end
|
|
328
|
-
|
|
329
497
|
# Map a daemon verdict string to a Result. The daemon reports the four
|
|
330
498
|
# run-time states it can decide; pre-fork states (skipped/no_coverage/…) are
|
|
331
499
|
# resolved tool-side before a request is ever sent.
|
|
@@ -346,6 +514,7 @@ module Mutineer
|
|
|
346
514
|
# tests drive directly: {boot_config} from the zero-dep suite and
|
|
347
515
|
# {build_coverage_map} from the daemon suite. Everything else is daemon-pipeline
|
|
348
516
|
# internals with no caller outside this file.
|
|
349
|
-
private_class_method :run_serial, :run_parallel, :job_result, :
|
|
517
|
+
private_class_method :sweep_daemon_temps, :run_serial, :run_parallel, :job_result, :result_for,
|
|
518
|
+
:prepare_run, :coverage_map_from_client, :worker_count_for
|
|
350
519
|
end
|
|
351
520
|
end
|
|
@@ -4,6 +4,8 @@ require "json"
|
|
|
4
4
|
require "io/wait"
|
|
5
5
|
require "open3"
|
|
6
6
|
require_relative "external_backend"
|
|
7
|
+
require_relative "rails_run_env"
|
|
8
|
+
require_relative "redaction"
|
|
7
9
|
|
|
8
10
|
module Mutineer
|
|
9
11
|
# Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
|
|
@@ -42,6 +44,9 @@ module Mutineer
|
|
|
42
44
|
# Seconds the daemon gets to boot the app and answer the handshake. A boot
|
|
43
45
|
# that never returns ends the run instead of hanging it (#101).
|
|
44
46
|
BOOT_TIMEOUT = 600
|
|
47
|
+
# Seconds close_io waits for the stderr drain to reach EOF before it stops
|
|
48
|
+
# the thread, so a last line without a newline is not lost.
|
|
49
|
+
DRAIN_GRACE = 1
|
|
45
50
|
# Bundler's marker for a variable that was unset before it activated.
|
|
46
51
|
BUNDLER_UNSET = "BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL"
|
|
47
52
|
|
|
@@ -71,6 +76,31 @@ module Mutineer
|
|
|
71
76
|
self
|
|
72
77
|
end
|
|
73
78
|
|
|
79
|
+
# The booted test environment's databases, as the daemon reported them:
|
|
80
|
+
# "names" and "primary", or "error" when the lookup failed. nil when the
|
|
81
|
+
# app has no worker-DB adapter.
|
|
82
|
+
#
|
|
83
|
+
# @return [Hash, nil]
|
|
84
|
+
attr_reader :databases
|
|
85
|
+
|
|
86
|
+
# Ask the daemon to create `slots` worker databases. Raises when the copy
|
|
87
|
+
# fails or another run holds the database, so the caller stops before any
|
|
88
|
+
# mutant is scored.
|
|
89
|
+
#
|
|
90
|
+
# @param slots [Integer] worker slots to create.
|
|
91
|
+
# @return [void]
|
|
92
|
+
# @raise [Mutineer::DaemonBootError]
|
|
93
|
+
def provision(slots)
|
|
94
|
+
raise DaemonBootError, "daemon is not running" if @stdin.nil?
|
|
95
|
+
|
|
96
|
+
send_line("cmd" => "provision", "slots" => slots)
|
|
97
|
+
reply = read_line(BOOT_TIMEOUT)
|
|
98
|
+
return if reply && reply["ok"]
|
|
99
|
+
|
|
100
|
+
detail = reply && reply["error"] ? reply["error"] : "provision failed"
|
|
101
|
+
raise DaemonBootError, detail
|
|
102
|
+
end
|
|
103
|
+
|
|
74
104
|
# Run one mutant: ship the payload + covering tests, return the verdict string.
|
|
75
105
|
# On a daemon crash (EOF/dead pipe) respawn (bounded) and return `"error"` for
|
|
76
106
|
# this mutant. Never a wrong verdict, never a wedged run.
|
|
@@ -150,6 +180,8 @@ module Mutineer
|
|
|
150
180
|
env["BUNDLE_GEMFILE"] = @gemfile
|
|
151
181
|
env["RBENV_VERSION"] = @ruby_version if @ruby_version
|
|
152
182
|
env["RAILS_ENV"] = "test" if rails_boot? && !env.key?("RAILS_ENV")
|
|
183
|
+
# The daemon loads tests after boot. Rails reads this when parallelize runs.
|
|
184
|
+
RailsRunEnv.pin_child!(env)
|
|
153
185
|
env
|
|
154
186
|
end
|
|
155
187
|
|
|
@@ -317,15 +349,14 @@ module Mutineer
|
|
|
317
349
|
"-r", DAEMON_PATH, "-e", "Mutineer::DaemonServer.run",
|
|
318
350
|
unsetenv_others: true, chdir: @app_root
|
|
319
351
|
)
|
|
320
|
-
#
|
|
321
|
-
#
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
end
|
|
352
|
+
# Replies are UTF-8 JSON, and the app's stderr is redacted as text: with
|
|
353
|
+
# LANG unset both pipes would read as US-ASCII (#303).
|
|
354
|
+
@stdout.set_encoding(Encoding::UTF_8)
|
|
355
|
+
@stderr.set_encoding(Encoding::UTF_8)
|
|
356
|
+
# Drain daemon stderr to the tool's stderr so child/boot errors are visible,
|
|
357
|
+
# redacted line by line (#288: the app's output goes through here too).
|
|
358
|
+
# Tracked (not fire-and-forget) so close_io can reclaim it on quit/respawn.
|
|
359
|
+
@drain = Thread.new { drain_stderr(@stderr) } # rubocop:disable ThreadSafety/NewThread
|
|
329
360
|
|
|
330
361
|
send_line(@boot)
|
|
331
362
|
read_line(BOOT_TIMEOUT)
|
|
@@ -344,6 +375,7 @@ module Mutineer
|
|
|
344
375
|
close_io
|
|
345
376
|
raise (timed_out ? DaemonBootTimeout : DaemonBootError), "daemon failed to boot under the app bundle: #{detail}"
|
|
346
377
|
end
|
|
378
|
+
@databases = ready["databases"]
|
|
347
379
|
end
|
|
348
380
|
|
|
349
381
|
# Respawn after a crash, up to MAX_RESTARTS, then hard-fail loudly.
|
|
@@ -383,6 +415,33 @@ module Mutineer
|
|
|
383
415
|
nil
|
|
384
416
|
end
|
|
385
417
|
|
|
418
|
+
# Copy each line of the daemon's stderr to the tool's stderr, redacted.
|
|
419
|
+
# A line is the unit the daemon and the app write, so a password is never
|
|
420
|
+
# split across two redaction calls. The line break is kept outside the
|
|
421
|
+
# redaction, so a line redaction hides as a whole still ends its line.
|
|
422
|
+
# After a write fails (EPIPE from a tool stderr that went away), the drain
|
|
423
|
+
# keeps reading and drops the lines, so the daemon never blocks on a full
|
|
424
|
+
# pipe. The outer rescue swallows the benign EBADF/IOError raised when
|
|
425
|
+
# close_io closes the pipe under an in-flight read, so the thread never
|
|
426
|
+
# ends with an error that close_io's join would re-raise.
|
|
427
|
+
#
|
|
428
|
+
# @param io [IO] the daemon's stderr pipe.
|
|
429
|
+
# @return [nil]
|
|
430
|
+
def drain_stderr(io)
|
|
431
|
+
writable = true
|
|
432
|
+
io.each_line do |line|
|
|
433
|
+
next unless writable
|
|
434
|
+
|
|
435
|
+
body = line.chomp("\n").scrub # an invalid byte must not stop the drain
|
|
436
|
+
@errio.write(Redaction.text(body), body.length < line.length ? "\n" : "")
|
|
437
|
+
rescue StandardError
|
|
438
|
+
writable = false
|
|
439
|
+
end
|
|
440
|
+
nil
|
|
441
|
+
rescue StandardError
|
|
442
|
+
nil
|
|
443
|
+
end
|
|
444
|
+
|
|
386
445
|
# Close the IPC pipes, stop the stderr-drain thread, and reap the daemon so a
|
|
387
446
|
# respawn or quit leaves no leaked fd, thread, or zombie.
|
|
388
447
|
#
|
|
@@ -395,7 +454,11 @@ module Mutineer
|
|
|
395
454
|
rescue Errno::ESRCH
|
|
396
455
|
nil # it exited between the check and the kill
|
|
397
456
|
end
|
|
398
|
-
|
|
457
|
+
# The drain holds a partial last line until EOF. Give it a moment to reach
|
|
458
|
+
# EOF now the daemon is dead, so a boot error with no final newline still
|
|
459
|
+
# prints; then stop it BEFORE closing its fd (avoids an EBADF mid-read).
|
|
460
|
+
@drain&.join(DRAIN_GRACE)
|
|
461
|
+
@drain&.kill
|
|
399
462
|
[@stdin, @stdout, @stderr].each { |io| io&.close rescue nil } # rubocop:disable Style/RescueModifier
|
|
400
463
|
@wait_thr&.join # reap the exited daemon so respawn/quit leaves no zombie
|
|
401
464
|
@stdin = @stdout = @stderr = @drain = @wait_thr = nil
|