mutineer 1.7.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +466 -0
- data/README.md +157 -747
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/cli.rb +132 -28
- data/lib/mutineer/concern_blocks.rb +187 -5
- data/lib/mutineer/config.rb +6 -0
- data/lib/mutineer/coverage_map.rb +92 -17
- data/lib/mutineer/daemon_backend.rb +75 -32
- data/lib/mutineer/daemon_client.rb +50 -17
- data/lib/mutineer/daemon_server.rb +93 -41
- data/lib/mutineer/external_backend.rb +150 -17
- data/lib/mutineer/init.rb +1 -1
- data/lib/mutineer/isolation.rb +70 -25
- data/lib/mutineer/job_plan.rb +119 -34
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +10 -8
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +22 -0
- data/lib/mutineer/mutators/call_to_receiver.rb +20 -7
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +18 -6
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +226 -12
- data/lib/mutineer/rails_worker_db.rb +200 -16
- data/lib/mutineer/redaction.rb +112 -11
- data/lib/mutineer/reporter.rb +109 -21
- data/lib/mutineer/result.rb +33 -5
- data/lib/mutineer/runner.rb +126 -18
- data/lib/mutineer/statement_lines.rb +14 -8
- data/lib/mutineer/subject.rb +13 -5
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners.rb +4 -2
- data/lib/mutineer/triage.rb +2 -2
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/skills/mutineer/SKILL.md +84 -0
- metadata +5 -1
|
@@ -28,12 +28,15 @@ module Mutineer
|
|
|
28
28
|
#
|
|
29
29
|
# Protocol (one JSON object per line, both directions):
|
|
30
30
|
# boot in : {"cmd":"boot","project_root":"...","boot":"config/environment","require_paths":[...],
|
|
31
|
-
# "load_paths":["test"],"framework":"minitest","rails":true
|
|
32
|
-
# ready out: {"ready":true,"ruby":"3.3.6","
|
|
31
|
+
# "load_paths":["test"],"framework":"minitest","rails":true}
|
|
32
|
+
# ready out: {"ready":true,"ruby":"3.3.6","databases":{"names":["primary"],"primary":"primary"}}
|
|
33
|
+
# ("databases" is null with no worker-DB adapter, {"error":"...","primary":"primary"|null} when the lookup fails)
|
|
33
34
|
# (or {"ready":false,"error":"..."} then exit)
|
|
34
35
|
# provision: {"cmd":"provision","slots":N} -> {"ok":true} or {"ok":false,"error":"..."}
|
|
35
36
|
# run in : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
|
|
36
37
|
# "tests":["test/models/order_test.rb"],"timeout":30}
|
|
38
|
+
# (a mutant of a def in a concern's hook block adds to the payload
|
|
39
|
+
# "hook_def":{"line":12,"name":"total","singleton":false}, #297)
|
|
37
40
|
# verdict : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
|
|
38
41
|
# quit in : {"cmd":"quit"}
|
|
39
42
|
#
|
|
@@ -45,29 +48,23 @@ module Mutineer
|
|
|
45
48
|
#
|
|
46
49
|
# Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
|
|
47
50
|
# failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
|
|
48
|
-
# routing failure); parent-detected timeout.
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
# implemented.
|
|
51
|
+
# routing failure); parent-detected timeout. Exit 2 also covers a run whose
|
|
52
|
+
# first failing test failed with an Active Record connection or lock error
|
|
53
|
+
# inside the test (#296): {MinitestIntegration::InTestDatabaseError} decides.
|
|
52
54
|
module DaemonServer
|
|
53
55
|
# Poll interval (seconds) for the per-fork deadline wait loop.
|
|
54
56
|
POLL = 0.02
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
# version
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
# Skips daemon mutant files when Zeitwerk scans an autoload directory.
|
|
62
|
-
# The boot sweep keeps a file another process still holds. Zeitwerk skips a
|
|
63
|
-
# hidden one (#253), but one an older version wrote has no dot, and it would
|
|
64
|
-
# stop the next boot, because its name is not a constant.
|
|
57
|
+
# Skips mutant files of both modes ({OrphanGuard::TEMP_BASENAME}) when
|
|
58
|
+
# Zeitwerk scans an autoload directory. The boot sweep keeps a file another
|
|
59
|
+
# process still holds. Zeitwerk skips a hidden one (#253), but one an older
|
|
60
|
+
# version wrote has no dot, and it would stop the next boot, because its
|
|
61
|
+
# name is not a constant.
|
|
65
62
|
module DaemonTempIgnore
|
|
66
63
|
# @return [void]
|
|
67
64
|
def setup
|
|
68
65
|
dirs = Array(Mutineer::DaemonServer.instance_variable_get(:@source_dirs))
|
|
69
66
|
dirs.each do |dir|
|
|
70
|
-
ignore(File.join(dir, Mutineer::
|
|
67
|
+
ignore(File.join(dir, Mutineer::OrphanGuard::TEMP_BASENAME))
|
|
71
68
|
end
|
|
72
69
|
super
|
|
73
70
|
end
|
|
@@ -87,11 +84,13 @@ module Mutineer
|
|
|
87
84
|
output ||= @protocol
|
|
88
85
|
@errio = errio
|
|
89
86
|
@output = output
|
|
87
|
+
# The client sends UTF-8 JSON; with LANG unset the pipe would read as US-ASCII (#303).
|
|
88
|
+
input.set_encoding(Encoding::UTF_8)
|
|
90
89
|
boot_line = input.gets
|
|
91
90
|
return if boot_line.nil? # client vanished before boot
|
|
92
91
|
|
|
93
92
|
boot!(JSON.parse(boot_line.strip))
|
|
94
|
-
output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION, "
|
|
93
|
+
output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION, "databases" => databases))
|
|
95
94
|
output.flush
|
|
96
95
|
|
|
97
96
|
input.each_line do |line|
|
|
@@ -197,8 +196,10 @@ module Mutineer
|
|
|
197
196
|
def setup_worker_db(cfg)
|
|
198
197
|
require_relative "rails_worker_db"
|
|
199
198
|
@worker_db = RailsWorkerDb.available? ? RailsWorkerDb : nil
|
|
200
|
-
|
|
201
|
-
@schema_path =
|
|
199
|
+
# Only the booted app knows its schema file (`schema_dump:` can rename it).
|
|
200
|
+
@schema_path = @worker_db&.ruby_schema_path
|
|
201
|
+
# Only a parallel worker daemon gets these names (DaemonBackend.boot_config).
|
|
202
|
+
@shared_databases = Array(cfg["shared_databases"]).map(&:to_s)
|
|
202
203
|
# Each SQLite slot is seeded once, on first use (not every mutant fork).
|
|
203
204
|
# Postgres and MySQL slots are copied at provision time, so this flag is unused there.
|
|
204
205
|
@slot_ready = {}
|
|
@@ -217,16 +218,44 @@ module Mutineer
|
|
|
217
218
|
@worker_db = nil
|
|
218
219
|
end
|
|
219
220
|
|
|
220
|
-
#
|
|
221
|
-
# worker
|
|
221
|
+
# The booted test environment's database config names and the one each
|
|
222
|
+
# worker copies, for the tool's worker count. nil when the app has no
|
|
223
|
+
# worker-DB adapter. A failed lookup is reported, not hidden: the tool
|
|
224
|
+
# then keeps one worker. That report still names the primary when it
|
|
225
|
+
# resolves, so the tool still rejects a `shared_databases` that lists it.
|
|
222
226
|
#
|
|
223
|
-
# @return [
|
|
224
|
-
def
|
|
225
|
-
return
|
|
227
|
+
# @return [Hash, nil]
|
|
228
|
+
def databases
|
|
229
|
+
return nil unless @worker_db
|
|
226
230
|
|
|
227
|
-
@worker_db.
|
|
231
|
+
@worker_db.database_names
|
|
232
|
+
rescue StandardError => e
|
|
233
|
+
{ "error" => describe_error(e), "primary" => primary_database_name }
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# The primary config name alone, or nil when it does not resolve either.
|
|
237
|
+
#
|
|
238
|
+
# @return [String, nil]
|
|
239
|
+
def primary_database_name
|
|
240
|
+
@worker_db.primary_database_name
|
|
228
241
|
rescue StandardError
|
|
229
|
-
|
|
242
|
+
nil
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Raise when a test in this fork used a database that parallel workers
|
|
246
|
+
# share (`shared_databases`). Its verdict may rest on another worker's
|
|
247
|
+
# rows, so the mutant is scored `error`. A serial run sends no names.
|
|
248
|
+
#
|
|
249
|
+
# @return [void]
|
|
250
|
+
# @raise [RuntimeError] when a shared database holds a connection.
|
|
251
|
+
def check_shared_databases!
|
|
252
|
+
return if @worker_db.nil? || @shared_databases.nil? || @shared_databases.empty?
|
|
253
|
+
|
|
254
|
+
used = @worker_db.touched_shared_databases(@shared_databases)
|
|
255
|
+
return if used.empty?
|
|
256
|
+
|
|
257
|
+
raise "shared database used during a parallel run: #{used.join(", ")}. " \
|
|
258
|
+
"Remove it from shared_databases, or see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#more-than-one-database"
|
|
230
259
|
end
|
|
231
260
|
|
|
232
261
|
# Run {RailsWorkerDb.provision} and report success or the cause. A failure
|
|
@@ -270,11 +299,13 @@ module Mutineer
|
|
|
270
299
|
|
|
271
300
|
# `Class: message` for an error the daemon reports, with credentials hidden.
|
|
272
301
|
# Rails parses DATABASE_URL itself, so its URI error can carry the password.
|
|
302
|
+
# {Redaction.describe} never raises, so a rescue that calls this cannot
|
|
303
|
+
# make the raw error the cause of a new one.
|
|
273
304
|
#
|
|
274
305
|
# @param error [Exception]
|
|
275
306
|
# @return [String]
|
|
276
307
|
def describe_error(error)
|
|
277
|
-
|
|
308
|
+
Redaction.describe(error)
|
|
278
309
|
end
|
|
279
310
|
|
|
280
311
|
# Hide database credentials in text bound for the client or stderr.
|
|
@@ -327,7 +358,12 @@ module Mutineer
|
|
|
327
358
|
# A routing failure raises here and is scored `error`, never a false verdict.
|
|
328
359
|
@worker_db&.after_fork(worker, first_use ? @schema_path : nil, seed: first_use)
|
|
329
360
|
apply_payload(req["payload"])
|
|
330
|
-
|
|
361
|
+
# A parallel worker records each pool that runs SQL, so a test
|
|
362
|
+
# that closes its shared connection is still caught below.
|
|
363
|
+
@worker_db.watch_sql if @worker_db && @shared_databases&.any?
|
|
364
|
+
result = run_tests(Array(req["tests"]))
|
|
365
|
+
check_shared_databases!
|
|
366
|
+
result
|
|
331
367
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
332
368
|
@errio.puts("[daemon-child] #{describe_error(e)}")
|
|
333
369
|
2
|
|
@@ -336,8 +372,12 @@ module Mutineer
|
|
|
336
372
|
end
|
|
337
373
|
verdict = wait_verdict(pid, timeout)
|
|
338
374
|
# Mark ready only when the child finished cleanly after seeding
|
|
339
|
-
# (killed/survived). Timeout can interrupt mid-copy; error
|
|
340
|
-
# failure
|
|
375
|
+
# (killed/survived). Timeout can interrupt mid-copy; error can be a
|
|
376
|
+
# routing failure or a database error inside a test (#296). Both leave
|
|
377
|
+
# the slot unready so the next fork seeds again. A shared-database
|
|
378
|
+
# error (check_shared_databases!) also exits 2, so it seeds again too:
|
|
379
|
+
# the exit code cannot tell it from a routing failure. That costs only
|
|
380
|
+
# time, in a run where that check then fails every mutant.
|
|
341
381
|
@slot_ready[worker] = true if first_use && %w[killed survived].include?(verdict)
|
|
342
382
|
# A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
|
|
343
383
|
# so it cannot outlive the run.
|
|
@@ -371,12 +411,13 @@ module Mutineer
|
|
|
371
411
|
Zeitwerk::Loader.prepend(DaemonTempIgnore)
|
|
372
412
|
end
|
|
373
413
|
|
|
374
|
-
# Remove orphaned mutant tempfiles from the source dirs
|
|
375
|
-
# SIGKILL path cannot run the child's ensure). Mirrors
|
|
376
|
-
# Skips a file another process still owns
|
|
414
|
+
# Remove orphaned mutant tempfiles of both modes from the source dirs
|
|
415
|
+
# (parent-side; the SIGKILL path cannot run the child's ensure). Mirrors
|
|
416
|
+
# JobPlan.sweep_orphans. Skips a file another process still owns
|
|
417
|
+
# ({OrphanGuard.mutant_file_in_use?}).
|
|
377
418
|
def sweep_temps
|
|
378
419
|
@source_dirs.to_a.each do |dir|
|
|
379
|
-
Dir.glob(File.join(dir,
|
|
420
|
+
Dir.glob(File.join(dir, OrphanGuard::TEMP_BASENAME)).each do |path|
|
|
380
421
|
next if OrphanGuard.mutant_file_in_use?(path)
|
|
381
422
|
|
|
382
423
|
File.unlink(path) rescue nil # rubocop:disable Style/RescueModifier
|
|
@@ -432,6 +473,9 @@ module Mutineer
|
|
|
432
473
|
# removes orphans and never touches a live file. Same path for reload
|
|
433
474
|
# (whole file) and redefine (wrapped snippet). The backend sends the whole
|
|
434
475
|
# file, so the concern blocks it stores replace the forgotten ones (#256).
|
|
476
|
+
# With a `"hook_def"`, the loaded file stored a mutated hook-block def, and
|
|
477
|
+
# {ConcernBlocks.place_hook_def} loads it into each class that has the
|
|
478
|
+
# method; it raises when none was stored, so the mutant scores `error` (#297).
|
|
435
479
|
def apply_payload(payload)
|
|
436
480
|
dir = File.dirname(File.expand_path(payload.fetch("source_file")))
|
|
437
481
|
Tempfile.create([".mutineer_daemon", ".rb"], dir) do |f|
|
|
@@ -440,19 +484,27 @@ module Mutineer
|
|
|
440
484
|
f.flock(File::LOCK_EX) rescue nil # rubocop:disable Style/RescueModifier
|
|
441
485
|
f.write(payload.fetch("code"))
|
|
442
486
|
f.flush
|
|
443
|
-
ConcernBlocks.
|
|
487
|
+
ConcernBlocks.load_mutant(payload.fetch("source_file"), f.path)
|
|
444
488
|
end
|
|
489
|
+
hook = payload["hook_def"] or return
|
|
490
|
+
|
|
491
|
+
ConcernBlocks.place_hook_def(source_file: payload.fetch("source_file"), line: hook.fetch("line"),
|
|
492
|
+
name: hook.fetch("name").to_sym, singleton: hook.fetch("singleton"))
|
|
445
493
|
end
|
|
446
494
|
|
|
447
495
|
# Load the covering test files and run them; 0 = all passed (survived),
|
|
448
|
-
# 1 = a failure/error (killed)
|
|
496
|
+
# 1 = a failure/error (killed), 2 = the first failure was an in-test
|
|
497
|
+
# database error (error, #296). Minitest only; rspec is not yet on this path.
|
|
498
|
+
# {MinitestIntegration.run} is the method the daemon's clean check uses,
|
|
499
|
+
# so each mutant fork gets the same seed and class list (#282). It loads
|
|
500
|
+
# only stdlib, so the daemon still needs no Prism. The fork requires
|
|
501
|
+
# nothing else first: the test files' `require "test_helper"` loads
|
|
502
|
+
# rails/test_help after the class list is reset, as in the clean check.
|
|
449
503
|
def run_tests(tests)
|
|
450
504
|
raise "unsupported framework #{@framework.inspect}" unless @framework == "minitest"
|
|
451
505
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
tests.each { |t| load File.expand_path(t) }
|
|
455
|
-
Minitest.run([]) ? 0 : 1
|
|
506
|
+
require_relative "minitest_integration"
|
|
507
|
+
MinitestIntegration.run(tests.map { |t| File.expand_path(t) })
|
|
456
508
|
end
|
|
457
509
|
end
|
|
458
510
|
end
|
|
@@ -4,6 +4,7 @@ require "shellwords"
|
|
|
4
4
|
require "tempfile"
|
|
5
5
|
require_relative "result"
|
|
6
6
|
require_relative "rails_run_env"
|
|
7
|
+
require_relative "redaction"
|
|
7
8
|
|
|
8
9
|
module Mutineer
|
|
9
10
|
# Raised when the smoke check (the unmutated suite) is not green, so the run
|
|
@@ -18,15 +19,18 @@ module Mutineer
|
|
|
18
19
|
# This is deliberately NOT a `TestRunners` framework adapter: those return an
|
|
19
20
|
# Integer 0/1 from inside a fork and are dispatched by framework name. This is a
|
|
20
21
|
# whole backend — it spawns a process, enforces a wall-clock timeout, and maps
|
|
21
|
-
# the exit status to a Result. The mapping
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
22
|
+
# the exit status to a Result. The mapping follows Isolation.decode: exit 0 is
|
|
23
|
+
# survived, exit 1 is killed, and any other exit is error (#292). When the clean
|
|
24
|
+
# smoke run printed a Minitest or RSpec summary line, exit 1 is a kill only when
|
|
25
|
+
# the output also shows a failing test, so a suite that crashed before its tests
|
|
26
|
+
# ran is not a kill. The smoke check (below) guards a persistently broken
|
|
27
|
+
# environment, and the score is disclosed as an upper bound.
|
|
25
28
|
#
|
|
26
29
|
# Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
|
|
27
30
|
# version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
|
|
28
31
|
# removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
|
|
29
|
-
# PARALLEL_WORKERS and DISABLE_SPRING are set to 1 ({RailsRunEnv})
|
|
32
|
+
# PARALLEL_WORKERS and DISABLE_SPRING are set to 1 ({RailsRunEnv}), and so is
|
|
33
|
+
# DISABLE_BOOTSNAP_COMPILE_CACHE ({child_env}).
|
|
30
34
|
module ExternalBackend
|
|
31
35
|
# Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
|
|
32
36
|
# the full suite). The per-mutant timeout is derived from how long this took.
|
|
@@ -37,6 +41,26 @@ module Mutineer
|
|
|
37
41
|
# Exit codes a shell or wrapper uses when the command could not run at all
|
|
38
42
|
# (126: not executable, 127: not found). Scored `error`, never `killed`.
|
|
39
43
|
CANNOT_RUN_CODES = [126, 127].freeze
|
|
44
|
+
# The exit code Minitest, `bin/rails test` and RSpec use for a failing test.
|
|
45
|
+
# Any other non-zero code is scored `error`, the same rule as Isolation.decode.
|
|
46
|
+
FAILURE_CODE = 1
|
|
47
|
+
|
|
48
|
+
# The end-of-run summary line of Minitest 5 and 6 ("N runs") and of
|
|
49
|
+
# minitest-reporters ("N tests"). Captures the failure and error counts.
|
|
50
|
+
MINITEST_SUMMARY = /\d+ (?:runs|tests), \d+ assertions, (\d+) failures, (\d+) errors/
|
|
51
|
+
# The end-of-run summary line of RSpec ("N examples, N failures").
|
|
52
|
+
# Captures the failure count.
|
|
53
|
+
RSPEC_SUMMARY = /\d+ examples?, (\d+) failures?/
|
|
54
|
+
# The line Minitest prints once every test file has loaded, before the first
|
|
55
|
+
# test runs. Exit 1 after it with no summary line means code under test
|
|
56
|
+
# called `exit 1` or `abort`, which the in-process backend also scores killed.
|
|
57
|
+
MINITEST_RUNNING = "# Running:"
|
|
58
|
+
# The line SimpleCov prints when its minimum_coverage gate fails the run.
|
|
59
|
+
SIMPLECOV_GATE = /SimpleCov failed with exit \d+ due to a coverage related error/
|
|
60
|
+
# RSpec's note that a file raised outside any example, such as at load.
|
|
61
|
+
RSPEC_OUTSIDE_ERROR = /errors? occurred outside of examples/
|
|
62
|
+
# ANSI colour codes, removed before matching so a coloured summary still counts.
|
|
63
|
+
ANSI_CODE = /\e\[[\d;]*m/
|
|
40
64
|
|
|
41
65
|
# PATH entries that pin a concrete Ruby under a version manager (ahead of
|
|
42
66
|
# shims). Optional trailing slash; normal bins are left alone.
|
|
@@ -72,8 +96,15 @@ module Mutineer
|
|
|
72
96
|
# onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
|
|
73
97
|
# injection and version-manager PATH pins so shims / `.ruby-version` can win.
|
|
74
98
|
# Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
|
|
75
|
-
# still reaches the suite. Also sets +PARALLEL_WORKERS+
|
|
76
|
-
# +
|
|
99
|
+
# still reaches the suite. Also sets +PARALLEL_WORKERS+, +DISABLE_SPRING+ and
|
|
100
|
+
# +DISABLE_BOOTSNAP_COMPILE_CACHE+ to 1. A wrapper can change them again.
|
|
101
|
+
# Bootsnap checks only that its variable is set, so only unsetting it turns
|
|
102
|
+
# the cache back on. The Bootsnap pin is here, not in
|
|
103
|
+
# {RailsRunEnv.pin_child!}: this backend rewrites each source file in place,
|
|
104
|
+
# and Bootsnap keys its compile cache on path, size and whole-second mtime,
|
|
105
|
+
# so a same-size mutant written in the same second can load the previous
|
|
106
|
+
# mutant's cached code (#285). The daemon loads each mutant from a new file
|
|
107
|
+
# and keeps the cache.
|
|
77
108
|
#
|
|
78
109
|
# @api private
|
|
79
110
|
# @return [Hash{String => String, nil}] env for Process.spawn.
|
|
@@ -97,6 +128,7 @@ module Mutineer
|
|
|
97
128
|
end
|
|
98
129
|
env["PATH"] = path.join(File::PATH_SEPARATOR)
|
|
99
130
|
RailsRunEnv.pin_child!(env, spring: true)
|
|
131
|
+
env["DISABLE_BOOTSNAP_COMPILE_CACHE"] = "1"
|
|
100
132
|
env
|
|
101
133
|
end
|
|
102
134
|
|
|
@@ -132,46 +164,142 @@ module Mutineer
|
|
|
132
164
|
# @param files [Array<String>] test file paths.
|
|
133
165
|
# @param timeout [Numeric] per-mutant wall-clock timeout in seconds.
|
|
134
166
|
# @param verbose [Boolean] print the child's captured output on a non-pass.
|
|
167
|
+
# @param check_summary [Boolean] score exit 1 as killed only when the output
|
|
168
|
+
# has a summary line that reports a failing test (see {failure_reported?}).
|
|
169
|
+
# The runner turns this on when the clean smoke run printed a summary line.
|
|
135
170
|
# @return [Mutineer::Result]
|
|
136
|
-
def self.run(command, files, timeout:, verbose: false)
|
|
171
|
+
def self.run(command, files, timeout:, verbose: false, check_summary: false)
|
|
137
172
|
kind, code, output, = spawn_capture(command, files, timeout)
|
|
138
173
|
case kind
|
|
139
174
|
when :timeout
|
|
140
175
|
# A timeout is the one non-pass we flag by default — a normal kill is also
|
|
141
176
|
# a non-zero exit, so notifying on every non-zero would spam every kill.
|
|
142
177
|
warn "[mutineer] test-command exceeded #{timeout}s and was killed (scored timeout)."
|
|
143
|
-
warn output if verbose && !output.empty?
|
|
178
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
144
179
|
Result.timeout
|
|
145
180
|
else # :exited
|
|
146
181
|
return Result.survived if code&.zero?
|
|
147
182
|
# Signal death (nil exitstatus) is infrastructure, not a suite assertion
|
|
148
183
|
# failure — match Isolation/daemon (error), do not inflate kill rate.
|
|
149
184
|
if code.nil?
|
|
150
|
-
warn output if verbose && !output.empty?
|
|
185
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
151
186
|
return Result.error("test-command terminated by signal")
|
|
152
187
|
end
|
|
153
188
|
|
|
154
|
-
warn output if verbose && !output.empty?
|
|
189
|
+
warn Redaction.text(output) if verbose && !output.empty?
|
|
155
190
|
# 126 (cannot execute) and 127 (command not found) come from a wrapper
|
|
156
191
|
# such as `bundle exec` that could not start the suite: no test ran.
|
|
157
192
|
return Result.error("test-command could not start (exit #{code})") if CANNOT_RUN_CODES.include?(code)
|
|
158
193
|
|
|
194
|
+
text = summary_text(output)
|
|
195
|
+
# SimpleCov's minimum_coverage gate replaced the exit code. Under
|
|
196
|
+
# Minitest it runs in Minitest.after_run even after a failing test, so a
|
|
197
|
+
# failing summary plus its message is still a kill. A passing summary
|
|
198
|
+
# plus its message means no test caught the mutant: in process, the
|
|
199
|
+
# fork exits before that hook, and the mutant survives. Keyed on
|
|
200
|
+
# SimpleCov's own message, so code under test that calls `exit 2`
|
|
201
|
+
# still scores error. Ceiling: with `SimpleCov.print_error_status =
|
|
202
|
+
# false` the gate prints nothing, and its exit scores error too.
|
|
203
|
+
if text.match?(SIMPLECOV_GATE)
|
|
204
|
+
return Result.killed if failure_reported?(text)
|
|
205
|
+
return Result.survived if tests_passed?(text)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
unless code == FAILURE_CODE
|
|
209
|
+
return Result.error("test-command exited #{code} (only exit 1 counts as a kill; " \
|
|
210
|
+
"check RSpec --failure-exit-code, a coverage gate, or your wrapper)")
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
if check_summary && !failure_reported?(text) && !exited_during_tests?(text)
|
|
214
|
+
return Result.error("test-command exited 1 but its output shows no failing test " \
|
|
215
|
+
"(the suite or the mutant crashed before tests ran; --verbose shows the output)")
|
|
216
|
+
end
|
|
217
|
+
|
|
159
218
|
Result.killed
|
|
160
219
|
end
|
|
161
220
|
end
|
|
162
221
|
|
|
222
|
+
# True when the text has a Minitest or RSpec summary line.
|
|
223
|
+
#
|
|
224
|
+
# @api private
|
|
225
|
+
# @param text [String] output from {summary_text}.
|
|
226
|
+
# @return [Boolean]
|
|
227
|
+
def self.summary_line?(text)
|
|
228
|
+
text.match?(MINITEST_SUMMARY) || text.match?(RSPEC_SUMMARY)
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# True when a summary line reports a failing test: a Minitest failure or
|
|
232
|
+
# error, or an RSpec failure. An RSpec error outside examples alone (a spec
|
|
233
|
+
# file that raised while loading) is not one, as in TestRunners::RSpec.
|
|
234
|
+
# Any matching line counts, so a wrapper that runs two suites still kills.
|
|
235
|
+
# A test that prints a fake summary line to stdout fools this check.
|
|
236
|
+
#
|
|
237
|
+
# @api private
|
|
238
|
+
# @param text [String] output from {summary_text}.
|
|
239
|
+
# @return [Boolean]
|
|
240
|
+
def self.failure_reported?(text)
|
|
241
|
+
text.scan(MINITEST_SUMMARY).any? { |failures, errors| (failures.to_i + errors.to_i).positive? } ||
|
|
242
|
+
text.scan(RSPEC_SUMMARY).any? { |(failures)| failures.to_i.positive? }
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# True when Minitest started its tests and the process then ended with no
|
|
246
|
+
# summary line after that start: code under test called `exit 1` or
|
|
247
|
+
# `abort` mid-run. Only the last start counts, so a wrapper whose earlier
|
|
248
|
+
# suite printed a summary still kills. A load crash ends before Minitest
|
|
249
|
+
# prints {MINITEST_RUNNING}, so it stays false.
|
|
250
|
+
# Only the default Minitest reporter prints that line: under RSpec or
|
|
251
|
+
# minitest-reporters, such an exit scores error, not killed. Ceiling: an
|
|
252
|
+
# exception that escapes Minitest.run itself (a reporter plugin bug) also
|
|
253
|
+
# reads as a kill here, where the in-process backend scores error.
|
|
254
|
+
#
|
|
255
|
+
# @api private
|
|
256
|
+
# @param text [String] output from {summary_text}.
|
|
257
|
+
# @return [Boolean]
|
|
258
|
+
def self.exited_during_tests?(text)
|
|
259
|
+
running_at = text.rindex(MINITEST_RUNNING)
|
|
260
|
+
return false unless running_at
|
|
261
|
+
|
|
262
|
+
summary_at = [text.rindex(MINITEST_SUMMARY), text.rindex(RSPEC_SUMMARY)].compact.max
|
|
263
|
+
summary_at.nil? || summary_at < running_at
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# True when a recognized summary line is present and none shows a failing
|
|
267
|
+
# test or an RSpec error outside examples. With no recognized summary line
|
|
268
|
+
# (a JUnit or custom reporter), it is false, so a SimpleCov gate exit there
|
|
269
|
+
# scores error rather than a survivor the output cannot prove.
|
|
270
|
+
#
|
|
271
|
+
# @api private
|
|
272
|
+
# @param text [String] output from {summary_text}.
|
|
273
|
+
# @return [Boolean]
|
|
274
|
+
def self.tests_passed?(text)
|
|
275
|
+
summary_line?(text) && !failure_reported?(text) && !text.match?(RSPEC_OUTSIDE_ERROR)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# The output as binary with ANSI colour codes removed. Binary, so bytes that
|
|
279
|
+
# are invalid in the default encoding (an unset locale) cannot raise.
|
|
280
|
+
# Computed once per run and passed to the predicates above.
|
|
281
|
+
#
|
|
282
|
+
# @api private
|
|
283
|
+
# @param output [String] captured child stdout+stderr.
|
|
284
|
+
# @return [String]
|
|
285
|
+
def self.summary_text(output)
|
|
286
|
+
output.b.gsub(ANSI_CODE, "")
|
|
287
|
+
end
|
|
288
|
+
|
|
163
289
|
# Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
|
|
164
|
-
# returns the elapsed seconds (used to calibrate the per-mutant timeout)
|
|
165
|
-
#
|
|
290
|
+
# returns the elapsed seconds (used to calibrate the per-mutant timeout) and
|
|
291
|
+
# whether the output has a Minitest or RSpec summary line (used to turn on
|
|
292
|
+
# the summary check in {run}); anything else raises SmokeCheckError so the
|
|
293
|
+
# run aborts before scoring.
|
|
166
294
|
#
|
|
167
295
|
# @param command [String] the --test-command template.
|
|
168
296
|
# @param files [Array<String>] test file paths.
|
|
169
297
|
# @param timeout [Numeric] ceiling for the calibration run.
|
|
170
|
-
# @return [Float]
|
|
298
|
+
# @return [Array(Float, Boolean)] `[elapsed_seconds, summary_seen]`.
|
|
171
299
|
# @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
|
|
172
300
|
def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
|
|
173
301
|
kind, code, output, elapsed = spawn_capture(command, files, timeout)
|
|
174
|
-
return elapsed if kind == :exited && code&.zero?
|
|
302
|
+
return [elapsed, summary_line?(summary_text(output))] if kind == :exited && code&.zero?
|
|
175
303
|
|
|
176
304
|
reason =
|
|
177
305
|
if kind == :timeout then "did not finish within #{timeout}s"
|
|
@@ -213,7 +341,7 @@ module Mutineer
|
|
|
213
341
|
"ASDF_RUBY_VERSION from the child env; the suite still picked the wrong " \
|
|
214
342
|
"Ruby. Use a wrapper that re-selects the app Ruby and ensure " \
|
|
215
343
|
".ruby-version / .tool-versions is present " \
|
|
216
|
-
"(see
|
|
344
|
+
"(see https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#apps-on-ruby--34, section Under a version manager)"
|
|
217
345
|
parts
|
|
218
346
|
end
|
|
219
347
|
|
|
@@ -245,7 +373,12 @@ module Mutineer
|
|
|
245
373
|
kind, code = wait_with_timeout(pid, timeout)
|
|
246
374
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
247
375
|
out.rewind
|
|
248
|
-
|
|
376
|
+
# Test output is arbitrary bytes. Read it as UTF-8 whatever the locale (#303).
|
|
377
|
+
# Each invalid byte becomes `\xNN`, as in Reporter#printable, so the
|
|
378
|
+
# smoke-failure checks can match on the text and nothing is lost.
|
|
379
|
+
output = out.read.force_encoding(Encoding::UTF_8)
|
|
380
|
+
.scrub { |bytes| bytes.unpack("C*").map { |b| format("\\x%02X", b) }.join }
|
|
381
|
+
[kind, code, output, elapsed]
|
|
249
382
|
ensure
|
|
250
383
|
if out
|
|
251
384
|
out.close
|