mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "tempfile"
|
|
5
5
|
require_relative "child_stdout"
|
|
6
|
+
require_relative "child_coverage"
|
|
6
7
|
require_relative "orphan_guard"
|
|
8
|
+
require_relative "concern_blocks"
|
|
9
|
+
require_relative "redaction"
|
|
7
10
|
|
|
8
11
|
module Mutineer
|
|
9
12
|
# App-side daemon (persistent worker).
|
|
@@ -25,28 +28,47 @@ module Mutineer
|
|
|
25
28
|
#
|
|
26
29
|
# Protocol (one JSON object per line, both directions):
|
|
27
30
|
# boot in : {"cmd":"boot","project_root":"...","boot":"config/environment","require_paths":[...],
|
|
28
|
-
# "load_paths":["test"],"framework":"minitest","rails":true
|
|
29
|
-
# ready out: {"ready":true,"ruby":"3.3.6"
|
|
31
|
+
# "load_paths":["test"],"framework":"minitest","rails":true}
|
|
32
|
+
# ready out: {"ready":true,"ruby":"3.3.6","databases":{"names":["primary"],"primary":"primary"}}
|
|
33
|
+
# ("databases" is null with no worker-DB adapter, {"error":"...","primary":"primary"|null} when the lookup fails)
|
|
34
|
+
# (or {"ready":false,"error":"..."} then exit)
|
|
35
|
+
# provision: {"cmd":"provision","slots":N} -> {"ok":true} or {"ok":false,"error":"..."}
|
|
30
36
|
# run in : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
|
|
31
37
|
# "tests":["test/models/order_test.rb"],"timeout":30}
|
|
38
|
+
# (a mutant of a def in a concern's hook block adds to the payload
|
|
39
|
+
# "hook_def":{"line":12,"name":"total","singleton":false}, #297)
|
|
32
40
|
# verdict : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
|
|
33
41
|
# quit in : {"cmd":"quit"}
|
|
34
42
|
#
|
|
35
43
|
# Worker isolation: when the app is Rails, each fork is routed to its own
|
|
36
|
-
# database
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
44
|
+
# database via {RailsWorkerDb} BEFORE any test loads, so concurrent workers
|
|
45
|
+
# cannot clobber each other's transactional fixtures. `worker` defaults to 0
|
|
46
|
+
# (serial). SQLite copies a file per slot. Postgres and MySQL use
|
|
47
|
+
# `<base>-mutineer-<w>`, created once before the worker daemons start.
|
|
40
48
|
#
|
|
41
49
|
# Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
|
|
42
50
|
# failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
|
|
43
|
-
# routing failure); parent-detected timeout.
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
# implemented.
|
|
51
|
+
# routing failure); parent-detected timeout. Exit 2 also covers a run whose
|
|
52
|
+
# first failing test failed with an Active Record connection or lock error
|
|
53
|
+
# inside the test (#296): {MinitestIntegration::InTestDatabaseError} decides.
|
|
47
54
|
module DaemonServer
|
|
48
55
|
# Poll interval (seconds) for the per-fork deadline wait loop.
|
|
49
56
|
POLL = 0.02
|
|
57
|
+
# Skips mutant files of both modes ({OrphanGuard::TEMP_BASENAME}) when
|
|
58
|
+
# Zeitwerk scans an autoload directory. The boot sweep keeps a file another
|
|
59
|
+
# process still holds. Zeitwerk skips a hidden one (#253), but one an older
|
|
60
|
+
# version wrote has no dot, and it would stop the next boot, because its
|
|
61
|
+
# name is not a constant.
|
|
62
|
+
module DaemonTempIgnore
|
|
63
|
+
# @return [void]
|
|
64
|
+
def setup
|
|
65
|
+
dirs = Array(Mutineer::DaemonServer.instance_variable_get(:@source_dirs))
|
|
66
|
+
dirs.each do |dir|
|
|
67
|
+
ignore(File.join(dir, Mutineer::OrphanGuard::TEMP_BASENAME))
|
|
68
|
+
end
|
|
69
|
+
super
|
|
70
|
+
end
|
|
71
|
+
end
|
|
50
72
|
|
|
51
73
|
class << self
|
|
52
74
|
# Serve the protocol on the given IO pair (defaults to stdio). Returns on quit.
|
|
@@ -62,11 +84,13 @@ module Mutineer
|
|
|
62
84
|
output ||= @protocol
|
|
63
85
|
@errio = errio
|
|
64
86
|
@output = output
|
|
87
|
+
# The client sends UTF-8 JSON; with LANG unset the pipe would read as US-ASCII (#303).
|
|
88
|
+
input.set_encoding(Encoding::UTF_8)
|
|
65
89
|
boot_line = input.gets
|
|
66
90
|
return if boot_line.nil? # client vanished before boot
|
|
67
91
|
|
|
68
92
|
boot!(JSON.parse(boot_line.strip))
|
|
69
|
-
output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION))
|
|
93
|
+
output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION, "databases" => databases))
|
|
70
94
|
output.flush
|
|
71
95
|
|
|
72
96
|
input.each_line do |line|
|
|
@@ -84,6 +108,14 @@ module Mutineer
|
|
|
84
108
|
end
|
|
85
109
|
break if req["cmd"] == "quit"
|
|
86
110
|
|
|
111
|
+
# Create worker databases before coverage forks or mutant forks.
|
|
112
|
+
# The first daemon does this. A restarted worker daemon does not.
|
|
113
|
+
if req["cmd"] == "provision"
|
|
114
|
+
output.puts(JSON.generate(provision_slots(req["slots"])))
|
|
115
|
+
output.flush
|
|
116
|
+
next
|
|
117
|
+
end
|
|
118
|
+
|
|
87
119
|
# Build the coverage map app-side and ship it to the tool, which then
|
|
88
120
|
# selects covering tests per mutant. One-shot control message.
|
|
89
121
|
if req["cmd"] == "coverage"
|
|
@@ -129,11 +161,17 @@ module Mutineer
|
|
|
129
161
|
require "coverage"
|
|
130
162
|
Coverage.start(lines: true, methods: true)
|
|
131
163
|
end
|
|
132
|
-
# Clear
|
|
133
|
-
#
|
|
134
|
-
#
|
|
164
|
+
# Clear a mutant tempfile a prior SIGKILLed child orphaned in a source
|
|
165
|
+
# dir BEFORE the app boots. Zeitwerk skips a hidden one, but it would
|
|
166
|
+
# choke on the non-constant name of one an older version wrote without
|
|
167
|
+
# the dot (#253). A file another
|
|
168
|
+
# live process still owns stays: this runs before the database lock,
|
|
169
|
+
# so a refused boot must not delete that run's mutant. The kept file
|
|
170
|
+
# is hidden from Zeitwerk instead.
|
|
135
171
|
sweep_temps
|
|
172
|
+
install_zeitwerk_ignore! if cfg["rails"]
|
|
136
173
|
require File.expand_path(cfg["boot"]) if cfg["boot"]
|
|
174
|
+
@zeitwerk_trace&.disable
|
|
137
175
|
# The --require files, after the boot as in-process (Runner.execute), so
|
|
138
176
|
# the coverage peek sees what they ran at load.
|
|
139
177
|
Array(cfg["require_paths"]).each { |f| require File.expand_path(f) }
|
|
@@ -141,7 +179,7 @@ module Mutineer
|
|
|
141
179
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
142
180
|
# Boot failed (bad boot path, app error). Tell the client and exit so it
|
|
143
181
|
# can surface a clean error rather than hang on the handshake.
|
|
144
|
-
@output.puts(JSON.generate("ready" => false, "error" =>
|
|
182
|
+
@output.puts(JSON.generate("ready" => false, "error" => describe_error(e)))
|
|
145
183
|
@output.flush
|
|
146
184
|
exit!(1)
|
|
147
185
|
end
|
|
@@ -149,20 +187,89 @@ module Mutineer
|
|
|
149
187
|
# Load the per-worker DB adapter app-side (sibling gem file, by relative path
|
|
150
188
|
# so it bypasses the app bundle, like this daemon itself). No-op unless the
|
|
151
189
|
# app has ActiveRecord. Records the adapter + schema path so each fork can
|
|
152
|
-
# route to its own database.
|
|
153
|
-
#
|
|
190
|
+
# route to its own database.
|
|
191
|
+
#
|
|
192
|
+
# `db_role` "prepare" waits for a provision command (the first daemon).
|
|
193
|
+
# "worker" only takes the shared lock (a worker daemon, including a
|
|
194
|
+
# respawn). Anything else provisions now, which is what a direct client
|
|
195
|
+
# that never sends the command needs. SQLite provisioning does nothing.
|
|
154
196
|
def setup_worker_db(cfg)
|
|
155
197
|
require_relative "rails_worker_db"
|
|
156
198
|
@worker_db = RailsWorkerDb.available? ? RailsWorkerDb : nil
|
|
157
|
-
|
|
158
|
-
@schema_path =
|
|
159
|
-
#
|
|
199
|
+
# Only the booted app knows its schema file (`schema_dump:` can rename it).
|
|
200
|
+
@schema_path = @worker_db&.ruby_schema_path
|
|
201
|
+
# Only a parallel worker daemon gets these names (DaemonBackend.boot_config).
|
|
202
|
+
@shared_databases = Array(cfg["shared_databases"]).map(&:to_s)
|
|
203
|
+
# Each SQLite slot is seeded once, on first use (not every mutant fork).
|
|
204
|
+
# Postgres and MySQL slots are copied at provision time, so this flag is unused there.
|
|
160
205
|
@slot_ready = {}
|
|
206
|
+
return unless @worker_db
|
|
207
|
+
|
|
208
|
+
case cfg["db_role"]
|
|
209
|
+
when "prepare"
|
|
210
|
+
nil
|
|
211
|
+
when "worker"
|
|
212
|
+
@worker_db.prepare_worker!
|
|
213
|
+
else
|
|
214
|
+
@worker_db.provision(cfg.fetch("slots", 2).to_i)
|
|
215
|
+
end
|
|
161
216
|
rescue LoadError => e
|
|
162
|
-
@errio.puts("[daemon] worker-DB routing unavailable: #{e.message}")
|
|
217
|
+
@errio.puts("[daemon] worker-DB routing unavailable: #{redact(e.message)}")
|
|
163
218
|
@worker_db = nil
|
|
164
219
|
end
|
|
165
220
|
|
|
221
|
+
# The booted test environment's database config names and the one each
|
|
222
|
+
# worker copies, for the tool's worker count. nil when the app has no
|
|
223
|
+
# worker-DB adapter. A failed lookup is reported, not hidden: the tool
|
|
224
|
+
# then keeps one worker. That report still names the primary when it
|
|
225
|
+
# resolves, so the tool still rejects a `shared_databases` that lists it.
|
|
226
|
+
#
|
|
227
|
+
# @return [Hash, nil]
|
|
228
|
+
def databases
|
|
229
|
+
return nil unless @worker_db
|
|
230
|
+
|
|
231
|
+
@worker_db.database_names
|
|
232
|
+
rescue StandardError => e
|
|
233
|
+
{ "error" => describe_error(e), "primary" => primary_database_name }
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# The primary config name alone, or nil when it does not resolve either.
|
|
237
|
+
#
|
|
238
|
+
# @return [String, nil]
|
|
239
|
+
def primary_database_name
|
|
240
|
+
@worker_db.primary_database_name
|
|
241
|
+
rescue StandardError
|
|
242
|
+
nil
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Raise when a test in this fork used a database that parallel workers
|
|
246
|
+
# share (`shared_databases`). Its verdict may rest on another worker's
|
|
247
|
+
# rows, so the mutant is scored `error`. A serial run sends no names.
|
|
248
|
+
#
|
|
249
|
+
# @return [void]
|
|
250
|
+
# @raise [RuntimeError] when a shared database holds a connection.
|
|
251
|
+
def check_shared_databases!
|
|
252
|
+
return if @worker_db.nil? || @shared_databases.nil? || @shared_databases.empty?
|
|
253
|
+
|
|
254
|
+
used = @worker_db.touched_shared_databases(@shared_databases)
|
|
255
|
+
return if used.empty?
|
|
256
|
+
|
|
257
|
+
raise "shared database used during a parallel run: #{used.join(", ")}. " \
|
|
258
|
+
"Remove it from shared_databases, or see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#more-than-one-database"
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Run {RailsWorkerDb.provision} and report success or the cause. A failure
|
|
262
|
+
# stays on this reply. It does not become a mutant verdict.
|
|
263
|
+
#
|
|
264
|
+
# @param slots [Integer, String]
|
|
265
|
+
# @return [Hash]
|
|
266
|
+
def provision_slots(slots)
|
|
267
|
+
@worker_db.provision(Integer(slots)) if @worker_db
|
|
268
|
+
{ "ok" => true }
|
|
269
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
270
|
+
{ "ok" => false, "error" => describe_error(e) }
|
|
271
|
+
end
|
|
272
|
+
|
|
166
273
|
# Build the coverage map app-side (Coverage was started at boot) and return
|
|
167
274
|
# it as `{map, failed_test_files, load_lines, load_methods}` for the tool to select covering tests.
|
|
168
275
|
# Capture forks route to worker 0's DB (isolated, serial). On any failure
|
|
@@ -185,8 +292,28 @@ module Mutineer
|
|
|
185
292
|
"failed_clean_tests" => cmap.failed_clean_tests, "load_lines" => cmap.load_lines.to_a,
|
|
186
293
|
"load_methods" => cmap.load_methods.to_a }
|
|
187
294
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
188
|
-
|
|
189
|
-
|
|
295
|
+
error = describe_error(e)
|
|
296
|
+
@errio.puts("[daemon] coverage build failed: #{error}")
|
|
297
|
+
{ "map" => {}, "failed_test_files" => [], "error" => error }
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# `Class: message` for an error the daemon reports, with credentials hidden.
|
|
301
|
+
# Rails parses DATABASE_URL itself, so its URI error can carry the password.
|
|
302
|
+
# {Redaction.describe} never raises, so a rescue that calls this cannot
|
|
303
|
+
# make the raw error the cause of a new one.
|
|
304
|
+
#
|
|
305
|
+
# @param error [Exception]
|
|
306
|
+
# @return [String]
|
|
307
|
+
def describe_error(error)
|
|
308
|
+
Redaction.describe(error)
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
# Hide database credentials in text bound for the client or stderr.
|
|
312
|
+
#
|
|
313
|
+
# @param text [String]
|
|
314
|
+
# @return [String]
|
|
315
|
+
def redact(text)
|
|
316
|
+
Redaction.text(text)
|
|
190
317
|
end
|
|
191
318
|
|
|
192
319
|
# Fork-safety hook for coverage capture. Each capture fork drops its copy
|
|
@@ -225,35 +352,75 @@ module Mutineer
|
|
|
225
352
|
code =
|
|
226
353
|
begin
|
|
227
354
|
ChildStdout.silence
|
|
355
|
+
ChildCoverage.allow_start
|
|
228
356
|
close_protocol
|
|
229
357
|
# Route THIS fork at its own worker database before any test loads.
|
|
230
358
|
# A routing failure raises here and is scored `error`, never a false verdict.
|
|
231
359
|
@worker_db&.after_fork(worker, first_use ? @schema_path : nil, seed: first_use)
|
|
232
360
|
apply_payload(req["payload"])
|
|
233
|
-
|
|
361
|
+
# A parallel worker records each pool that runs SQL, so a test
|
|
362
|
+
# that closes its shared connection is still caught below.
|
|
363
|
+
@worker_db.watch_sql if @worker_db && @shared_databases&.any?
|
|
364
|
+
result = run_tests(Array(req["tests"]))
|
|
365
|
+
check_shared_databases!
|
|
366
|
+
result
|
|
234
367
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
235
|
-
@errio.puts("[daemon-child] #{e
|
|
368
|
+
@errio.puts("[daemon-child] #{describe_error(e)}")
|
|
236
369
|
2
|
|
237
370
|
end
|
|
238
371
|
exit!(code)
|
|
239
372
|
end
|
|
240
373
|
verdict = wait_verdict(pid, timeout)
|
|
241
374
|
# Mark ready only when the child finished cleanly after seeding
|
|
242
|
-
# (killed/survived). Timeout can interrupt mid-copy; error
|
|
243
|
-
# failure
|
|
375
|
+
# (killed/survived). Timeout can interrupt mid-copy; error can be a
|
|
376
|
+
# routing failure or a database error inside a test (#296). Both leave
|
|
377
|
+
# the slot unready so the next fork seeds again. A shared-database
|
|
378
|
+
# error (check_shared_databases!) also exits 2, so it seeds again too:
|
|
379
|
+
# the exit code cannot tell it from a routing failure. That costs only
|
|
380
|
+
# time, in a run where that check then fails every mutant.
|
|
244
381
|
@slot_ready[worker] = true if first_use && %w[killed survived].include?(verdict)
|
|
245
382
|
# A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
|
|
246
|
-
# so it cannot outlive the run
|
|
383
|
+
# so it cannot outlive the run.
|
|
247
384
|
sweep_temps if verdict == "timeout"
|
|
248
385
|
{ "id" => req["id"], "verdict" => verdict }
|
|
249
386
|
end
|
|
250
387
|
|
|
251
|
-
#
|
|
252
|
-
#
|
|
388
|
+
# Hide kept mutant files from Zeitwerk before the app builds its loaders.
|
|
389
|
+
# The loader class may not exist yet. Watch for it, then stop watching.
|
|
390
|
+
#
|
|
391
|
+
# @return [void]
|
|
392
|
+
def install_zeitwerk_ignore!
|
|
393
|
+
if defined?(Zeitwerk::Loader)
|
|
394
|
+
prepend_daemon_temp_ignore
|
|
395
|
+
return
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
@zeitwerk_trace = TracePoint.new(:end) do |event|
|
|
399
|
+
next unless event.self.is_a?(Class) && event.self.name == "Zeitwerk::Loader"
|
|
400
|
+
|
|
401
|
+
@zeitwerk_trace.disable
|
|
402
|
+
prepend_daemon_temp_ignore
|
|
403
|
+
end
|
|
404
|
+
@zeitwerk_trace.enable
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
# @return [void]
|
|
408
|
+
def prepend_daemon_temp_ignore
|
|
409
|
+
return if Zeitwerk::Loader.ancestors.include?(DaemonTempIgnore)
|
|
410
|
+
|
|
411
|
+
Zeitwerk::Loader.prepend(DaemonTempIgnore)
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# Remove orphaned mutant tempfiles of both modes from the source dirs
|
|
415
|
+
# (parent-side; the SIGKILL path cannot run the child's ensure). Mirrors
|
|
416
|
+
# JobPlan.sweep_orphans. Skips a file another process still owns
|
|
417
|
+
# ({OrphanGuard.mutant_file_in_use?}).
|
|
253
418
|
def sweep_temps
|
|
254
419
|
@source_dirs.to_a.each do |dir|
|
|
255
|
-
Dir.glob(File.join(dir,
|
|
256
|
-
|
|
420
|
+
Dir.glob(File.join(dir, OrphanGuard::TEMP_BASENAME)).each do |path|
|
|
421
|
+
next if OrphanGuard.mutant_file_in_use?(path)
|
|
422
|
+
|
|
423
|
+
File.unlink(path) rescue nil # rubocop:disable Style/RescueModifier
|
|
257
424
|
end
|
|
258
425
|
end
|
|
259
426
|
end
|
|
@@ -300,27 +467,44 @@ module Mutineer
|
|
|
300
467
|
# `require_relative` in the mutated source resolves against its real
|
|
301
468
|
# neighbours. Writing it to the tmpdir would LoadError on such files and
|
|
302
469
|
# score a spurious `error` that diverges from the in-process path. The
|
|
303
|
-
#
|
|
304
|
-
#
|
|
305
|
-
#
|
|
470
|
+
# leading dot keeps the file away from Zeitwerk, which skips hidden files:
|
|
471
|
+
# loading it can be the first time Zeitwerk scans a namespace directory,
|
|
472
|
+
# and a parallel worker's file may sit there too (#253). `sweep_temps` only
|
|
473
|
+
# removes orphans and never touches a live file. Same path for reload
|
|
474
|
+
# (whole file) and redefine (wrapped snippet). The backend sends the whole
|
|
475
|
+
# file, so the concern blocks it stores replace the forgotten ones (#256).
|
|
476
|
+
# With a `"hook_def"`, the loaded file stored a mutated hook-block def, and
|
|
477
|
+
# {ConcernBlocks.place_hook_def} loads it into each class that has the
|
|
478
|
+
# method; it raises when none was stored, so the mutant scores `error` (#297).
|
|
306
479
|
def apply_payload(payload)
|
|
307
480
|
dir = File.dirname(File.expand_path(payload.fetch("source_file")))
|
|
308
|
-
Tempfile.create(["mutineer_daemon", ".rb"], dir) do |f|
|
|
481
|
+
Tempfile.create([".mutineer_daemon", ".rb"], dir) do |f|
|
|
482
|
+
# Held until this block ends, including SIGKILL (the kernel drops it).
|
|
483
|
+
# Another run's sweep sees the lock and does not delete the file.
|
|
484
|
+
f.flock(File::LOCK_EX) rescue nil # rubocop:disable Style/RescueModifier
|
|
309
485
|
f.write(payload.fetch("code"))
|
|
310
486
|
f.flush
|
|
311
|
-
load f.path
|
|
487
|
+
ConcernBlocks.replace(payload.fetch("source_file")) { load f.path }
|
|
312
488
|
end
|
|
489
|
+
hook = payload["hook_def"] or return
|
|
490
|
+
|
|
491
|
+
ConcernBlocks.place_hook_def(source_file: payload.fetch("source_file"), line: hook.fetch("line"),
|
|
492
|
+
name: hook.fetch("name").to_sym, singleton: hook.fetch("singleton"))
|
|
313
493
|
end
|
|
314
494
|
|
|
315
495
|
# Load the covering test files and run them; 0 = all passed (survived),
|
|
316
|
-
# 1 = a failure/error (killed)
|
|
496
|
+
# 1 = a failure/error (killed), 2 = the first failure was an in-test
|
|
497
|
+
# database error (error, #296). Minitest only; rspec is not yet on this path.
|
|
498
|
+
# {MinitestIntegration.run} is the method the daemon's clean check uses,
|
|
499
|
+
# so each mutant fork gets the same seed and class list (#282). It loads
|
|
500
|
+
# only stdlib, so the daemon still needs no Prism. The fork requires
|
|
501
|
+
# nothing else first: the test files' `require "test_helper"` loads
|
|
502
|
+
# rails/test_help after the class list is reset, as in the clean check.
|
|
317
503
|
def run_tests(tests)
|
|
318
504
|
raise "unsupported framework #{@framework.inspect}" unless @framework == "minitest"
|
|
319
505
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
tests.each { |t| load File.expand_path(t) }
|
|
323
|
-
Minitest.run([]) ? 0 : 1
|
|
506
|
+
require_relative "minitest_integration"
|
|
507
|
+
MinitestIntegration.run(tests.map { |t| File.expand_path(t) })
|
|
324
508
|
end
|
|
325
509
|
end
|
|
326
510
|
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require "shellwords"
|
|
4
4
|
require "tempfile"
|
|
5
5
|
require_relative "result"
|
|
6
|
+
require_relative "rails_run_env"
|
|
7
|
+
require_relative "redaction"
|
|
6
8
|
|
|
7
9
|
module Mutineer
|
|
8
10
|
# Raised when the smoke check (the unmutated suite) is not green, so the run
|
|
@@ -17,14 +19,18 @@ module Mutineer
|
|
|
17
19
|
# This is deliberately NOT a `TestRunners` framework adapter: those return an
|
|
18
20
|
# Integer 0/1 from inside a fork and are dispatched by framework name. This is a
|
|
19
21
|
# whole backend — it spawns a process, enforces a wall-clock timeout, and maps
|
|
20
|
-
# the exit status to a Result. The mapping
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
22
|
+
# the exit status to a Result. The mapping follows Isolation.decode: exit 0 is
|
|
23
|
+
# survived, exit 1 is killed, and any other exit is error (#292). When the clean
|
|
24
|
+
# smoke run printed a Minitest or RSpec summary line, exit 1 is a kill only when
|
|
25
|
+
# the output also shows a failing test, so a suite that crashed before its tests
|
|
26
|
+
# ran is not a kill. The smoke check (below) guards a persistently broken
|
|
27
|
+
# environment, and the score is disclosed as an upper bound.
|
|
24
28
|
#
|
|
25
29
|
# Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
|
|
26
30
|
# version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
|
|
27
31
|
# removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
|
|
32
|
+
# PARALLEL_WORKERS and DISABLE_SPRING are set to 1 ({RailsRunEnv}), and so is
|
|
33
|
+
# DISABLE_BOOTSNAP_COMPILE_CACHE ({child_env}).
|
|
28
34
|
module ExternalBackend
|
|
29
35
|
# Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
|
|
30
36
|
# the full suite). The per-mutant timeout is derived from how long this took.
|
|
@@ -32,6 +38,29 @@ module Mutineer
|
|
|
32
38
|
# Poll interval for the deadline wait loop. Independent of Isolation's loop —
|
|
33
39
|
# this backend waits on an external process TREE, not an in-process fork.
|
|
34
40
|
POLL = 0.02
|
|
41
|
+
# Exit codes a shell or wrapper uses when the command could not run at all
|
|
42
|
+
# (126: not executable, 127: not found). Scored `error`, never `killed`.
|
|
43
|
+
CANNOT_RUN_CODES = [126, 127].freeze
|
|
44
|
+
# The exit code Minitest, `bin/rails test` and RSpec use for a failing test.
|
|
45
|
+
# Any other non-zero code is scored `error`, the same rule as Isolation.decode.
|
|
46
|
+
FAILURE_CODE = 1
|
|
47
|
+
|
|
48
|
+
# The end-of-run summary line of Minitest 5 and 6 ("N runs") and of
|
|
49
|
+
# minitest-reporters ("N tests"). Captures the failure and error counts.
|
|
50
|
+
MINITEST_SUMMARY = /\d+ (?:runs|tests), \d+ assertions, (\d+) failures, (\d+) errors/
|
|
51
|
+
# The end-of-run summary line of RSpec ("N examples, N failures").
|
|
52
|
+
# Captures the failure count.
|
|
53
|
+
RSPEC_SUMMARY = /\d+ examples?, (\d+) failures?/
|
|
54
|
+
# The line Minitest prints once every test file has loaded, before the first
|
|
55
|
+
# test runs. Exit 1 after it with no summary line means code under test
|
|
56
|
+
# called `exit 1` or `abort`, which the in-process backend also scores killed.
|
|
57
|
+
MINITEST_RUNNING = "# Running:"
|
|
58
|
+
# The line SimpleCov prints when its minimum_coverage gate fails the run.
|
|
59
|
+
SIMPLECOV_GATE = /SimpleCov failed with exit \d+ due to a coverage related error/
|
|
60
|
+
# RSpec's note that a file raised outside any example, such as at load.
|
|
61
|
+
RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
|
|
62
|
+
# ANSI colour codes, removed before matching so a coloured summary still counts.
|
|
63
|
+
ANSI_CODE = /\e\[[\d;]*m/
|
|
35
64
|
|
|
36
65
|
# PATH entries that pin a concrete Ruby under a version manager (ahead of
|
|
37
66
|
# shims). Optional trailing slash; normal bins are left alone.
|
|
@@ -67,7 +96,15 @@ module Mutineer
|
|
|
67
96
|
# onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
|
|
68
97
|
# injection and version-manager PATH pins so shims / `.ruby-version` can win.
|
|
69
98
|
# Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
|
|
70
|
-
# still reaches the suite.
|
|
99
|
+
# still reaches the suite. Also sets +PARALLEL_WORKERS+, +DISABLE_SPRING+ and
|
|
100
|
+
# +DISABLE_BOOTSNAP_COMPILE_CACHE+ to 1. A wrapper can change them again.
|
|
101
|
+
# Bootsnap checks only that its variable is set, so only unsetting it turns
|
|
102
|
+
# the cache back on. The Bootsnap pin is here, not in
|
|
103
|
+
# {RailsRunEnv.pin_child!}: this backend rewrites each source file in place,
|
|
104
|
+
# and Bootsnap keys its compile cache on path, size and whole-second mtime,
|
|
105
|
+
# so a same-size mutant written in the same second can load the previous
|
|
106
|
+
# mutant's cached code (#285). The daemon loads each mutant from a new file
|
|
107
|
+
# and keeps the cache.
|
|
71
108
|
#
|
|
72
109
|
# @api private
|
|
73
110
|
# @return [Hash{String => String, nil}] env for Process.spawn.
|
|
@@ -90,6 +127,8 @@ module Mutineer
|
|
|
90
127
|
kept
|
|
91
128
|
end
|
|
92
129
|
env["PATH"] = path.join(File::PATH_SEPARATOR)
|
|
130
|
+
RailsRunEnv.pin_child!(env, spring: true)
|
|
131
|
+
env["DISABLE_BOOTSNAP_COMPILE_CACHE"] = "1"
|
|
93
132
|
env
|
|
94
133
|
end
|
|
95
134
|
|
|
@@ -125,42 +164,142 @@ module Mutineer
|
|
|
125
164
|
# @param files [Array<String>] test file paths.
|
|
126
165
|
# @param timeout [Numeric] per-mutant wall-clock timeout in seconds.
|
|
127
166
|
# @param verbose [Boolean] print the child's captured output on a non-pass.
|
|
167
|
+
# @param check_summary [Boolean] score exit 1 as killed only when the output
|
|
168
|
+
# has a summary line that reports a failing test (see {failure_reported?}).
|
|
169
|
+
# The runner turns this on when the clean smoke run printed a summary line.
|
|
128
170
|
# @return [Mutineer::Result]
|
|
129
|
-
def self.run(command, files, timeout:, verbose: false)
|
|
171
|
+
def self.run(command, files, timeout:, verbose: false, check_summary: false)
|
|
130
172
|
kind, code, output, = spawn_capture(command, files, timeout)
|
|
131
173
|
case kind
|
|
132
174
|
when :timeout
|
|
133
175
|
# A timeout is the one non-pass we flag by default — a normal kill is also
|
|
134
176
|
# a non-zero exit, so notifying on every non-zero would spam every kill.
|
|
135
177
|
warn "[mutineer] test-command exceeded #{timeout}s and was killed (scored timeout)."
|
|
136
|
-
warn output if verbose && !output.empty?
|
|
178
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
137
179
|
Result.timeout
|
|
138
180
|
else # :exited
|
|
139
181
|
return Result.survived if code&.zero?
|
|
140
182
|
# Signal death (nil exitstatus) is infrastructure, not a suite assertion
|
|
141
183
|
# failure — match Isolation/daemon (error), do not inflate kill rate.
|
|
142
184
|
if code.nil?
|
|
143
|
-
warn output if verbose && !output.empty?
|
|
185
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
144
186
|
return Result.error("test-command terminated by signal")
|
|
145
187
|
end
|
|
146
188
|
|
|
147
|
-
warn output if verbose && !output.empty?
|
|
189
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
190
|
+
# 126 (cannot execute) and 127 (command not found) come from a wrapper
|
|
191
|
+
# such as `bundle exec` that could not start the suite: no test ran.
|
|
192
|
+
return Result.error("test-command could not start (exit #{code})") if CANNOT_RUN_CODES.include?(code)
|
|
193
|
+
|
|
194
|
+
text = summary_text(output)
|
|
195
|
+
# SimpleCov's minimum_coverage gate replaced the exit code. Under
|
|
196
|
+
# Minitest it runs in Minitest.after_run even after a failing test, so a
|
|
197
|
+
# failing summary plus its message is still a kill. A passing summary
|
|
198
|
+
# plus its message means no test caught the mutant: in process, the
|
|
199
|
+
# fork exits before that hook, and the mutant survives. Keyed on
|
|
200
|
+
# SimpleCov's own message, so code under test that calls `exit 2`
|
|
201
|
+
# still scores error. Ceiling: with `SimpleCov.print_error_status =
|
|
202
|
+
# false` the gate prints nothing, and its exit scores error too.
|
|
203
|
+
if text.match?(SIMPLECOV_GATE)
|
|
204
|
+
return Result.killed if failure_reported?(text)
|
|
205
|
+
return Result.survived if tests_passed?(text)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
unless code == FAILURE_CODE
|
|
209
|
+
return Result.error("test-command exited #{code} (only exit 1 counts as a kill; " \
|
|
210
|
+
"check RSpec --failure-exit-code, a coverage gate, or your wrapper)")
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
if check_summary && !failure_reported?(text) && !exited_during_tests?(text)
|
|
214
|
+
return Result.error("test-command exited 1 but its output shows no failing test " \
|
|
215
|
+
"(the suite or the mutant crashed before tests ran; --verbose shows the output)")
|
|
216
|
+
end
|
|
217
|
+
|
|
148
218
|
Result.killed
|
|
149
219
|
end
|
|
150
220
|
end
|
|
151
221
|
|
|
222
|
+
# True when the text has a Minitest or RSpec summary line.
|
|
223
|
+
#
|
|
224
|
+
# @api private
|
|
225
|
+
# @param text [String] output from {summary_text}.
|
|
226
|
+
# @return [Boolean]
|
|
227
|
+
def self.summary_line?(text)
|
|
228
|
+
text.match?(MINITEST_SUMMARY) || text.match?(RSPEC_SUMMARY)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# True when a summary line reports a failing test: a Minitest failure or
|
|
232
|
+
# error, or an RSpec failure. An RSpec error outside examples alone (a spec
|
|
233
|
+
# file that raised while loading) is not one, as in TestRunners::RSpec.
|
|
234
|
+
# Any matching line counts, so a wrapper that runs two suites still kills.
|
|
235
|
+
# A test that prints a fake summary line to stdout fools this check.
|
|
236
|
+
#
|
|
237
|
+
# @api private
|
|
238
|
+
# @param text [String] output from {summary_text}.
|
|
239
|
+
# @return [Boolean]
|
|
240
|
+
def self.failure_reported?(text)
|
|
241
|
+
text.scan(MINITEST_SUMMARY).any? { |failures, errors| (failures.to_i + errors.to_i).positive? } ||
|
|
242
|
+
text.scan(RSPEC_SUMMARY).any? { |(failures)| failures.to_i.positive? }
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# True when Minitest started its tests and the process then ended with no
|
|
246
|
+
# summary line after that start: code under test called `exit 1` or
|
|
247
|
+
# `abort` mid-run. Only the last start counts, so a wrapper whose earlier
|
|
248
|
+
# suite printed a summary still kills. A load crash ends before Minitest
|
|
249
|
+
# prints {MINITEST_RUNNING}, so it stays false.
|
|
250
|
+
# Only the default Minitest reporter prints that line: under RSpec or
|
|
251
|
+
# minitest-reporters, such an exit scores error, not killed. Ceiling: an
|
|
252
|
+
# exception that escapes Minitest.run itself (a reporter plugin bug) also
|
|
253
|
+
# reads as a kill here, where the in-process backend scores error.
|
|
254
|
+
#
|
|
255
|
+
# @api private
|
|
256
|
+
# @param text [String] output from {summary_text}.
|
|
257
|
+
# @return [Boolean]
|
|
258
|
+
def self.exited_during_tests?(text)
|
|
259
|
+
running_at = text.rindex(MINITEST_RUNNING)
|
|
260
|
+
return false unless running_at
|
|
261
|
+
|
|
262
|
+
summary_at = [text.rindex(MINITEST_SUMMARY), text.rindex(RSPEC_SUMMARY)].compact.max
|
|
263
|
+
summary_at.nil? || summary_at < running_at
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# True when a recognized summary line is present and none shows a failing
|
|
267
|
+
# test or an RSpec error outside examples. With no recognized summary line
|
|
268
|
+
# (a JUnit or custom reporter), it is false, so a SimpleCov gate exit there
|
|
269
|
+
# scores error rather than a survivor the output cannot prove.
|
|
270
|
+
#
|
|
271
|
+
# @api private
|
|
272
|
+
# @param text [String] output from {summary_text}.
|
|
273
|
+
# @return [Boolean]
|
|
274
|
+
def self.tests_passed?(text)
|
|
275
|
+
summary_line?(text) && !failure_reported?(text) && !text.match?(RSPEC_OUTSIDE_ERROR)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# The output as binary with ANSI colour codes removed. Binary, so bytes that
|
|
279
|
+
# are invalid in the default encoding (an unset locale) cannot raise.
|
|
280
|
+
# Computed once per run and passed to the predicates above.
|
|
281
|
+
#
|
|
282
|
+
# @api private
|
|
283
|
+
# @param output [String] captured child stdout+stderr.
|
|
284
|
+
# @return [String]
|
|
285
|
+
def self.summary_text(output)
|
|
286
|
+
output.b.gsub(ANSI_CODE, "")
|
|
287
|
+
end
|
|
288
|
+
|
|
152
289
|
# Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
|
|
153
|
-
# returns the elapsed seconds (used to calibrate the per-mutant timeout)
|
|
154
|
-
#
|
|
290
|
+
# returns the elapsed seconds (used to calibrate the per-mutant timeout) and
|
|
291
|
+
# whether the output has a Minitest or RSpec summary line (used to turn on
|
|
292
|
+
# the summary check in {run}); anything else raises SmokeCheckError so the
|
|
293
|
+
# run aborts before scoring.
|
|
155
294
|
#
|
|
156
295
|
# @param command [String] the --test-command template.
|
|
157
296
|
# @param files [Array<String>] test file paths.
|
|
158
297
|
# @param timeout [Numeric] ceiling for the calibration run.
|
|
159
|
-
# @return [Float]
|
|
298
|
+
# @return [Array(Float, Boolean)] `[elapsed_seconds, summary_seen]`.
|
|
160
299
|
# @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
|
|
161
300
|
def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
|
|
162
301
|
kind, code, output, elapsed = spawn_capture(command, files, timeout)
|
|
163
|
-
return elapsed if kind == :exited && code&.zero?
|
|
302
|
+
return [elapsed, summary_line?(summary_text(output))] if kind == :exited && code&.zero?
|
|
164
303
|
|
|
165
304
|
reason =
|
|
166
305
|
if kind == :timeout then "did not finish within #{timeout}s"
|
|
@@ -202,7 +341,7 @@ module Mutineer
|
|
|
202
341
|
"ASDF_RUBY_VERSION from the child env; the suite still picked the wrong " \
|
|
203
342
|
"Ruby. Use a wrapper that re-selects the app Ruby and ensure " \
|
|
204
343
|
".ruby-version / .tool-versions is present " \
|
|
205
|
-
"(see
|
|
344
|
+
"(see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#apps-on-ruby--34, section Under a version manager)"
|
|
206
345
|
parts
|
|
207
346
|
end
|
|
208
347
|
|
|
@@ -234,7 +373,12 @@ module Mutineer
|
|
|
234
373
|
kind, code = wait_with_timeout(pid, timeout)
|
|
235
374
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
236
375
|
out.rewind
|
|
237
|
-
|
|
376
|
+
# Test output is arbitrary bytes. Read it as UTF-8 whatever the locale (#303).
|
|
377
|
+
# Each invalid byte becomes `\xNN`, as in Reporter#printable, so the
|
|
378
|
+
# smoke-failure checks can match on the text and nothing is lost.
|
|
379
|
+
output = out.read.force_encoding(Encoding::UTF_8)
|
|
380
|
+
.scrub { |bytes| bytes.unpack("C*").map { |b| format("\\x%02X", b) }.join }
|
|
381
|
+
[kind, code, output, elapsed]
|
|
238
382
|
ensure
|
|
239
383
|
if out
|
|
240
384
|
out.close
|