mutineer 0.11.1 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +55 -0
- data/README.md +39 -4
- data/lib/mutineer/baseline.rb +14 -16
- data/lib/mutineer/cli.rb +39 -41
- data/lib/mutineer/config.rb +17 -19
- data/lib/mutineer/coverage_map.rb +54 -52
- data/lib/mutineer/daemon_backend.rb +316 -0
- data/lib/mutineer/daemon_client.rb +60 -36
- data/lib/mutineer/daemon_server.rb +65 -61
- data/lib/mutineer/external_backend.rb +119 -11
- data/lib/mutineer/file_swap.rb +16 -15
- data/lib/mutineer/rails_worker_db.rb +44 -43
- data/lib/mutineer/reporter.rb +29 -31
- data/lib/mutineer/result.rb +25 -26
- data/lib/mutineer/runner.rb +69 -284
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +23 -23
- metadata +2 -1
|
@@ -4,19 +4,23 @@ require "json"
|
|
|
4
4
|
require "open3"
|
|
5
5
|
|
|
6
6
|
module Mutineer
|
|
7
|
-
# Raised when the daemon cannot be booted
|
|
8
|
-
#
|
|
7
|
+
# Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
|
|
8
|
+
# app error, a failed handshake, a spawn the OS refused, or MAX_RESTARTS crashes.
|
|
9
|
+
# It means "stop the run" — a backend that scored the remaining mutants against a
|
|
10
|
+
# dead daemon would report a score covering a fraction of the work. The CLI maps it
|
|
11
|
+
# to a runtime error (exit 1).
|
|
9
12
|
class DaemonBootError < StandardError; end
|
|
10
13
|
|
|
11
|
-
#
|
|
14
|
+
# Tool-side handle for the app-side daemon.
|
|
12
15
|
#
|
|
13
|
-
# Spawns `daemon_server.rb` UNDER THE APP'S BUNDLE/RUBY (cleaned env so the
|
|
14
|
-
# bundler context never leaks; the daemon file is loaded by absolute path
|
|
15
|
-
# `-r`, which bypasses the app bundle that has no mutineer), completes the
|
|
16
|
-
# handshake, then ships per-mutant payloads and reads structured verdicts.
|
|
17
|
-
# daemon dies mid-run it respawns (bounded) and marks the in-flight
|
|
18
|
-
# rather than corrupting the run. Reuses the cleaned-env spawn
|
|
19
|
-
# in the spike driver and the spawn discipline of
|
|
16
|
+
# Spawns `daemon_server.rb` UNDER THE APP'S BUNDLE/RUBY (cleaned env so the
|
|
17
|
+
# gem's bundler context never leaks; the daemon file is loaded by absolute path
|
|
18
|
+
# with `-r`, which bypasses the app bundle that has no mutineer), completes the
|
|
19
|
+
# ready handshake, then ships per-mutant payloads and reads structured verdicts.
|
|
20
|
+
# If the daemon dies mid-run it respawns (bounded) and marks the in-flight
|
|
21
|
+
# mutant `error` rather than corrupting the run. Reuses the cleaned-env spawn
|
|
22
|
+
# and stderr-drain proven in the spike driver and the spawn discipline of
|
|
23
|
+
# ExternalBackend.
|
|
20
24
|
class DaemonClient
|
|
21
25
|
# Absolute path to the daemon entry, loaded app-side by `-r` (bypasses the bundle).
|
|
22
26
|
DAEMON_PATH = File.expand_path("daemon_server.rb", __dir__)
|
|
@@ -49,16 +53,22 @@ module Mutineer
|
|
|
49
53
|
|
|
50
54
|
# Run one mutant: ship the payload + covering tests, return the verdict string.
|
|
51
55
|
# On a daemon crash (EOF/dead pipe) respawn (bounded) and return `"error"` for
|
|
52
|
-
# this mutant
|
|
56
|
+
# this mutant. Never a wrong verdict, never a wedged run.
|
|
53
57
|
#
|
|
54
58
|
# @param id [Integer] request id (echoed back for ordering safety).
|
|
55
59
|
# @param payload [Hash] {"code" => mutated ruby, "source_file" => path}.
|
|
56
60
|
# @param tests [Array<String>] covering test file paths.
|
|
57
61
|
# @param timeout [Numeric] per-mutant wall-clock timeout (seconds).
|
|
58
62
|
# @param worker [Integer] worker slot; the daemon routes the fork to
|
|
59
|
-
# `<db>-<worker>`
|
|
63
|
+
# `<db>-<worker>` for isolation. Defaults to 0 (serial).
|
|
60
64
|
# @return [String] one of survived/killed/error/timeout.
|
|
61
65
|
def request(id:, payload:, tests:, timeout:, worker: 0)
|
|
66
|
+
# close_io nils the pipes, so a client whose respawn never completed would
|
|
67
|
+
# otherwise fail per-mutant forever (NoMethodError on nil) and let the backend
|
|
68
|
+
# score every remaining mutant against nothing. Deadness is a property of the
|
|
69
|
+
# client, not of whichever exception happened to escape.
|
|
70
|
+
raise DaemonBootError, "daemon is not running" if @stdin.nil?
|
|
71
|
+
|
|
62
72
|
# A crash can surface on the WRITE (daemon died idle between requests →
|
|
63
73
|
# Errno::EPIPE) as well as the read (EOF), so guard both: either way, respawn
|
|
64
74
|
# for future mutants and score THIS one error (re-running a crash-causing
|
|
@@ -76,10 +86,11 @@ module Mutineer
|
|
|
76
86
|
"error"
|
|
77
87
|
end
|
|
78
88
|
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
# (possibly with an `"error"`), or nil if the daemon vanished
|
|
82
|
-
# falls back to running the full test set (no narrowing) rather than
|
|
89
|
+
# Ask the daemon to build the coverage map app-side and return it. One-shot
|
|
90
|
+
# control message (no id). Returns `{"map"=>..., "failed_test_files"=>...}`
|
|
91
|
+
# (possibly with an `"error"`), or nil if the daemon vanished. The caller then
|
|
92
|
+
# falls back to running the full test set (no narrowing) rather than
|
|
93
|
+
# mis-scoring.
|
|
83
94
|
#
|
|
84
95
|
# @return [Hash, nil] the coverage payload, or nil on a dead pipe.
|
|
85
96
|
def coverage
|
|
@@ -103,8 +114,8 @@ module Mutineer
|
|
|
103
114
|
|
|
104
115
|
private
|
|
105
116
|
|
|
106
|
-
# Cleaned environment for the app bundle: strip the gem's bundler/Ruby context
|
|
107
|
-
# `bundle exec` resolves the APP's Gemfile under the requested Ruby.
|
|
117
|
+
# Cleaned environment for the app bundle: strip the gem's bundler/Ruby context
|
|
118
|
+
# so `bundle exec` resolves the APP's Gemfile under the requested Ruby.
|
|
108
119
|
def app_env
|
|
109
120
|
env = ENV.to_h.reject { |k, _| k.start_with?("BUNDLE_", "RUBY", "GEM_") }
|
|
110
121
|
env["BUNDLE_GEMFILE"] = @gemfile
|
|
@@ -118,25 +129,38 @@ module Mutineer
|
|
|
118
129
|
# @return [void]
|
|
119
130
|
# @raise [Mutineer::DaemonBootError] when the daemon fails to boot.
|
|
120
131
|
def spawn_daemon
|
|
121
|
-
# Plain `bundle exec ruby
|
|
122
|
-
# non-rbenv setup. When bundler/ruby are rbenv shims, the RBENV_VERSION
|
|
123
|
-
# in app_env still selects the app's Ruby; otherwise the active
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
)
|
|
128
|
-
#
|
|
129
|
-
#
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
132
|
+
# Plain `bundle exec ruby`, NOT `rbenv exec`, which would break CI and any
|
|
133
|
+
# non-rbenv setup. When bundler/ruby are rbenv shims, the RBENV_VERSION
|
|
134
|
+
# carried in app_env still selects the app's Ruby; otherwise the active
|
|
135
|
+
# Ruby is used.
|
|
136
|
+
# Everything up to the handshake is terminal, not one mutant's problem: a spawn
|
|
137
|
+
# the OS refuses (EMFILE/ENOMEM under --jobs N, ENOENT when `bundle` does not
|
|
138
|
+
# resolve) and a daemon that dies before accepting the boot payload (EPIPE on
|
|
139
|
+
# the write) both leave a client that cannot recover. Raise the class that ends
|
|
140
|
+
# the run — a SystemCallError would reach the CLI as a usage error (exit 2).
|
|
141
|
+
ready =
|
|
142
|
+
begin
|
|
143
|
+
@stdin, @stdout, @stderr, @wait_thr = Open3.popen3(
|
|
144
|
+
app_env, "bundle", "exec", "ruby",
|
|
145
|
+
"-r", DAEMON_PATH, "-e", "Mutineer::DaemonServer.run", chdir: @app_root
|
|
146
|
+
)
|
|
147
|
+
# Drain daemon stderr to the tool's stderr so child/boot errors are visible.
|
|
148
|
+
# Tracked (not fire-and-forget) so close_io can reclaim it on quit/respawn;
|
|
149
|
+
# the rescue swallows the benign EBADF/IOError raised when close_io closes
|
|
150
|
+
# the pipe out from under an in-flight copy_stream.
|
|
151
|
+
@drain = Thread.new do # rubocop:disable ThreadSafety/NewThread
|
|
152
|
+
IO.copy_stream(@stderr, @errio)
|
|
153
|
+
rescue IOError, Errno::EBADF
|
|
154
|
+
nil
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
send_line(@boot)
|
|
158
|
+
read_line
|
|
159
|
+
rescue SystemCallError, IOError => e
|
|
160
|
+
close_io
|
|
161
|
+
raise DaemonBootError, "daemon could not be started: #{e.class}: #{e.message}"
|
|
162
|
+
end
|
|
137
163
|
|
|
138
|
-
send_line(@boot)
|
|
139
|
-
ready = read_line
|
|
140
164
|
unless ready && ready["ready"]
|
|
141
165
|
detail = ready && ready["error"] ? ready["error"] : "daemon exited before the handshake"
|
|
142
166
|
close_io
|
|
@@ -4,19 +4,20 @@ require "json"
|
|
|
4
4
|
require "tempfile"
|
|
5
5
|
|
|
6
6
|
module Mutineer
|
|
7
|
-
#
|
|
7
|
+
# App-side daemon (persistent worker).
|
|
8
8
|
#
|
|
9
9
|
# Runs UNDER THE APP'S OWN BUNDLE/RUBY (the tool's DaemonClient spawns it via
|
|
10
10
|
# `bundle exec ruby`). It boots the app ONCE, then serves per-mutant test-run
|
|
11
|
-
# requests over stdin/stdout as newline-delimited JSON. For each request it
|
|
12
|
-
# a child that loads the mutated source text the tool sent, runs the
|
|
13
|
-
# tests, and exits with a status the parent decodes into a verdict.
|
|
11
|
+
# requests over stdin/stdout as newline-delimited JSON. For each request it
|
|
12
|
+
# FORKS a child that loads the mutated source text the tool sent, runs the
|
|
13
|
+
# covering tests, and exits with a status the parent decodes into a verdict.
|
|
14
14
|
#
|
|
15
|
-
# HARD CONSTRAINT
|
|
16
|
-
#
|
|
17
|
-
# mutineer. So it requires ONLY stdlib + the app's own boot file; it
|
|
18
|
-
# the fork/timeout/decode loop rather than requiring
|
|
19
|
-
# Prism). All parsing/mutation happened
|
|
15
|
+
# HARD CONSTRAINT: this file must be loadable WITHOUT Prism or the rest of
|
|
16
|
+
# mutineer. The app's Ruby may be < 3.4 (no stdlib Prism) and its bundle has no
|
|
17
|
+
# mutineer. So it requires ONLY stdlib + the app's own boot file; it
|
|
18
|
+
# re-implements the fork/timeout/decode loop rather than requiring
|
|
19
|
+
# `isolation.rb` (which pulls in Prism). All parsing/mutation happened
|
|
20
|
+
# tool-side; the daemon only `load`s text.
|
|
20
21
|
#
|
|
21
22
|
# Protocol (one JSON object per line, both directions):
|
|
22
23
|
# boot in : {"cmd":"boot","project_root":"...","boot":"config/environment",
|
|
@@ -27,16 +28,18 @@ module Mutineer
|
|
|
27
28
|
# verdict : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
|
|
28
29
|
# quit in : {"cmd":"quit"}
|
|
29
30
|
#
|
|
30
|
-
# Worker isolation
|
|
31
|
-
# database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so
|
|
32
|
-
# workers
|
|
33
|
-
# (serial). SQLite this pass; Postgres provisioning is
|
|
31
|
+
# Worker isolation: when the app is Rails, each fork is routed to its own
|
|
32
|
+
# database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so
|
|
33
|
+
# concurrent workers cannot clobber each other's transactional fixtures.
|
|
34
|
+
# `worker` defaults to 0 (serial). SQLite this pass; Postgres provisioning is
|
|
35
|
+
# not yet implemented.
|
|
34
36
|
#
|
|
35
|
-
# Verdict mapping
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
# `killed` is
|
|
37
|
+
# Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
|
|
38
|
+
# failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
|
|
39
|
+
# routing failure); parent-detected timeout. Tagging an in-test DB error (one
|
|
40
|
+
# fired inside a test body, vs at routing time) as `error` rather than
|
|
41
|
+
# `killed` is only observable under the concurrent gate and is not yet
|
|
42
|
+
# implemented.
|
|
40
43
|
module DaemonServer
|
|
41
44
|
# Poll interval (seconds) for the per-fork deadline wait loop.
|
|
42
45
|
POLL = 0.02
|
|
@@ -65,16 +68,16 @@ module Mutineer
|
|
|
65
68
|
begin
|
|
66
69
|
req = JSON.parse(line)
|
|
67
70
|
rescue JSON::ParserError => e
|
|
68
|
-
# A corrupt line has no id to address a reply to (and the client only
|
|
69
|
-
# sends valid JSON, so it
|
|
70
|
-
# rather than write an unaddressable verdict onto the channel.
|
|
71
|
+
# A corrupt line has no id to address a reply to (and the client only
|
|
72
|
+
# ever sends valid JSON, so it cannot be a pending request). Log and
|
|
73
|
+
# read on rather than write an unaddressable verdict onto the channel.
|
|
71
74
|
@errio.puts("[daemon] dropped unparseable line: #{e.message}")
|
|
72
75
|
next
|
|
73
76
|
end
|
|
74
77
|
break if req["cmd"] == "quit"
|
|
75
78
|
|
|
76
|
-
#
|
|
77
|
-
#
|
|
79
|
+
# Build the coverage map app-side and ship it to the tool, which then
|
|
80
|
+
# selects covering tests per mutant. One-shot control message.
|
|
78
81
|
if req["cmd"] == "coverage"
|
|
79
82
|
output.puts(JSON.generate(build_coverage_map))
|
|
80
83
|
output.flush
|
|
@@ -88,8 +91,8 @@ module Mutineer
|
|
|
88
91
|
|
|
89
92
|
private
|
|
90
93
|
|
|
91
|
-
# BOOT ONCE. chdir + require the app's boot file so the whole app is loaded
|
|
92
|
-
# inherited by every fork. Never requires mutineer.
|
|
94
|
+
# BOOT ONCE. chdir + require the app's boot file so the whole app is loaded
|
|
95
|
+
# and inherited by every fork. Never requires mutineer.
|
|
93
96
|
def boot!(cfg)
|
|
94
97
|
@cfg = cfg
|
|
95
98
|
@framework = cfg.fetch("framework", "minitest")
|
|
@@ -97,30 +100,30 @@ module Mutineer
|
|
|
97
100
|
Dir.chdir(cfg["project_root"]) if cfg["project_root"]
|
|
98
101
|
ENV["RAILS_ENV"] ||= "test" if cfg["rails"]
|
|
99
102
|
Array(cfg["load_paths"]).each { |d| $LOAD_PATH.unshift(File.expand_path(d)) }
|
|
100
|
-
#
|
|
101
|
-
# instrumented
|
|
103
|
+
# Start Coverage BEFORE the app loads, so booted source lines are
|
|
104
|
+
# instrumented. The map build (build_via_fork) forks this booted parent.
|
|
102
105
|
if cfg["coverage"]
|
|
103
106
|
require "coverage"
|
|
104
107
|
Coverage.start(lines: true)
|
|
105
108
|
end
|
|
106
109
|
# Clear any mutant tempfile a prior SIGKILLed timeout child orphaned in a
|
|
107
|
-
# source dir BEFORE the app boots
|
|
110
|
+
# source dir BEFORE the app boots. Zeitwerk would otherwise choke on the
|
|
108
111
|
# tempfile's non-constant name during autoload setup.
|
|
109
112
|
sweep_temps
|
|
110
113
|
require File.expand_path(cfg["boot"]) if cfg["boot"]
|
|
111
114
|
setup_worker_db(cfg) if cfg["rails"]
|
|
112
115
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
113
|
-
# Boot failed (bad boot path, app error)
|
|
114
|
-
# surface a clean error rather than hang on the handshake.
|
|
116
|
+
# Boot failed (bad boot path, app error). Tell the client and exit so it
|
|
117
|
+
# can surface a clean error rather than hang on the handshake.
|
|
115
118
|
@output.puts(JSON.generate("ready" => false, "error" => "#{e.class}: #{e.message}"))
|
|
116
119
|
@output.flush
|
|
117
120
|
exit!(1)
|
|
118
121
|
end
|
|
119
122
|
|
|
120
|
-
# Load the per-worker DB adapter app-side (sibling gem file, by relative path
|
|
121
|
-
# it bypasses the app bundle
|
|
122
|
-
# ActiveRecord. Records the adapter + schema path so each fork can
|
|
123
|
-
#
|
|
123
|
+
# Load the per-worker DB adapter app-side (sibling gem file, by relative path
|
|
124
|
+
# so it bypasses the app bundle, like this daemon itself). No-op unless the
|
|
125
|
+
# app has ActiveRecord. Records the adapter + schema path so each fork can
|
|
126
|
+
# route to its own database. SQLite-only this pass; a non-SQLite config
|
|
124
127
|
# raises in the fork and reads as `error`, never a mis-routed verdict.
|
|
125
128
|
def setup_worker_db(cfg)
|
|
126
129
|
require_relative "rails_worker_db"
|
|
@@ -134,11 +137,11 @@ module Mutineer
|
|
|
134
137
|
@worker_db = nil
|
|
135
138
|
end
|
|
136
139
|
|
|
137
|
-
#
|
|
138
|
-
#
|
|
139
|
-
# Capture forks route to worker 0's DB (isolated, serial). On any failure
|
|
140
|
-
# an empty map + an error string
|
|
141
|
-
# rather than mis-scoring everything as no_coverage.
|
|
140
|
+
# Build the coverage map app-side (Coverage was started at boot) and return
|
|
141
|
+
# it as `{map, failed_test_files}` for the tool to select covering tests.
|
|
142
|
+
# Capture forks route to worker 0's DB (isolated, serial). On any failure
|
|
143
|
+
# return an empty map + an error string. The tool then falls back to the
|
|
144
|
+
# full test set rather than mis-scoring everything as no_coverage.
|
|
142
145
|
def build_coverage_map
|
|
143
146
|
require_relative "coverage_map"
|
|
144
147
|
root = @cfg["project_root"] || Dir.pwd
|
|
@@ -153,9 +156,9 @@ module Mutineer
|
|
|
153
156
|
{ "map" => {}, "failed_test_files" => [], "error" => "#{e.class}: #{e.message}" }
|
|
154
157
|
end
|
|
155
158
|
|
|
156
|
-
# Fork-safety hook for coverage capture: route each capture fork to worker
|
|
157
|
-
# isolated DB (captures run serially, so one worker is enough). Nil when
|
|
158
|
-
# has no worker-DB adapter (non-Rails)
|
|
159
|
+
# Fork-safety hook for coverage capture: route each capture fork to worker
|
|
160
|
+
# 0's isolated DB (captures run serially, so one worker is enough). Nil when
|
|
161
|
+
# the app has no worker-DB adapter (non-Rails). Capture then runs as before.
|
|
159
162
|
def coverage_after_fork
|
|
160
163
|
return nil unless @worker_db
|
|
161
164
|
|
|
@@ -190,16 +193,16 @@ module Mutineer
|
|
|
190
193
|
verdict = wait_verdict(pid, timeout)
|
|
191
194
|
# Mark ready only when the child finished cleanly after schema load
|
|
192
195
|
# (killed/survived). Timeout can interrupt mid-load_schema; error is a
|
|
193
|
-
# routing failure
|
|
196
|
+
# routing failure. Both leave the slot unready so the next fork reloads.
|
|
194
197
|
@schema_ready[worker] = true if schema_for_fork && %w[killed survived].include?(verdict)
|
|
195
|
-
# A SIGKILLed timeout child skipped its Tempfile unlink
|
|
196
|
-
# it
|
|
198
|
+
# A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
|
|
199
|
+
# so it cannot outlive the run or trip Zeitwerk on a later fork.
|
|
197
200
|
sweep_temps if verdict == "timeout"
|
|
198
201
|
{ "id" => req["id"], "verdict" => verdict }
|
|
199
202
|
end
|
|
200
203
|
|
|
201
204
|
# Remove orphaned mutant tempfiles from the source dirs (parent-side; the
|
|
202
|
-
# SIGKILL path
|
|
205
|
+
# SIGKILL path cannot run the child's ensure). Mirrors Runner.sweep_orphans.
|
|
203
206
|
def sweep_temps
|
|
204
207
|
@source_dirs.to_a.each do |dir|
|
|
205
208
|
Dir.glob(File.join(dir, "mutineer_daemon*.rb")).each do |f|
|
|
@@ -209,12 +212,12 @@ module Mutineer
|
|
|
209
212
|
end
|
|
210
213
|
|
|
211
214
|
# Single-waiter deadline loop (mirrors Isolation.run and
|
|
212
|
-
# ExternalBackend.wait_with_timeout, re-implemented here because Isolation
|
|
213
|
-
# in Prism which is forbidden app-side). NOTE: this is the 3rd copy of
|
|
214
|
-
# waitpid2(WNOHANG)+deadline+pgroup-SIGKILL+decode discipline
|
|
215
|
-
# kill/reap/decode logic must be applied to all three in lockstep.
|
|
216
|
-
# child's process group past the deadline; a signalled child
|
|
217
|
-
# `error`.
|
|
215
|
+
# ExternalBackend.wait_with_timeout, re-implemented here because Isolation
|
|
216
|
+
# pulls in Prism which is forbidden app-side). NOTE: this is the 3rd copy of
|
|
217
|
+
# the waitpid2(WNOHANG)+deadline+pgroup-SIGKILL+decode discipline. A fix to
|
|
218
|
+
# the kill/reap/decode logic must be applied to all three in lockstep.
|
|
219
|
+
# SIGKILL the child's process group past the deadline; a signalled child
|
|
220
|
+
# (nil exitstatus) is `error`.
|
|
218
221
|
def wait_verdict(pid, timeout)
|
|
219
222
|
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
|
220
223
|
loop do
|
|
@@ -242,14 +245,15 @@ module Mutineer
|
|
|
242
245
|
end
|
|
243
246
|
end
|
|
244
247
|
|
|
245
|
-
# Write the tool-built mutated text beside the real source and `load` it
|
|
246
|
-
# reopening the mutated class/method in THIS child only. It goes in the
|
|
247
|
-
# file's directory (like Isolation.apply_whole_file) so a
|
|
248
|
-
# the mutated source resolves against its real
|
|
249
|
-
# tmpdir would LoadError on such files and
|
|
250
|
-
# diverges from the in-process path. The
|
|
251
|
-
# autoload dir) is handled by the
|
|
252
|
-
# the file. Same path for
|
|
248
|
+
# Write the tool-built mutated text beside the real source and `load` it,
|
|
249
|
+
# reopening the mutated class/method in THIS child only. It goes in the
|
|
250
|
+
# source file's directory (like Isolation.apply_whole_file) so a
|
|
251
|
+
# `require_relative` in the mutated source resolves against its real
|
|
252
|
+
# neighbours. Writing it to the tmpdir would LoadError on such files and
|
|
253
|
+
# score a spurious `error` that diverges from the in-process path. The
|
|
254
|
+
# Zeitwerk hazard (a stray `.rb` in an autoload dir) is handled by the
|
|
255
|
+
# boot/timeout `sweep_temps`, not by relocating the file. Same path for
|
|
256
|
+
# reload (whole file) and redefine (wrapped snippet).
|
|
253
257
|
def apply_payload(payload)
|
|
254
258
|
dir = File.dirname(File.expand_path(payload.fetch("source_file")))
|
|
255
259
|
Tempfile.create(["mutineer_daemon", ".rb"], dir) do |f|
|
|
@@ -260,7 +264,7 @@ module Mutineer
|
|
|
260
264
|
end
|
|
261
265
|
|
|
262
266
|
# Load the covering test files and run them; 0 = all passed (survived),
|
|
263
|
-
# 1 = a failure/error (killed). Minitest only
|
|
267
|
+
# 1 = a failure/error (killed). Minitest only; rspec is not yet on this path.
|
|
264
268
|
def run_tests(tests)
|
|
265
269
|
raise "unsupported framework #{@framework.inspect}" unless @framework == "minitest"
|
|
266
270
|
|
|
@@ -10,17 +10,21 @@ module Mutineer
|
|
|
10
10
|
# tests. The CLI maps this to a runtime error (exit 1), not a usage error.
|
|
11
11
|
class SmokeCheckError < StandardError; end
|
|
12
12
|
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
13
|
+
# External execution backend. Runs the user's `--test-command` as a subprocess
|
|
14
|
+
# in the app's own runtime (whatever Ruby its bundle resolves to), so mutineer
|
|
15
|
+
# (Ruby ≥ 3.4) can mutation-test apps pinned to an older Ruby.
|
|
16
16
|
#
|
|
17
17
|
# This is deliberately NOT a `TestRunners` framework adapter: those return an
|
|
18
18
|
# Integer 0/1 from inside a fork and are dispatched by framework name. This is a
|
|
19
19
|
# whole backend — it spawns a process, enforces a wall-clock timeout, and maps
|
|
20
|
-
# the exit status to a Result. The mapping is the
|
|
20
|
+
# the exit status to a Result. The mapping is the same direction as in-process
|
|
21
21
|
# (suite passes => survived, suite fails => killed) but coarser: it cannot tell an
|
|
22
22
|
# infrastructure error from a genuine kill, so the smoke check (below) guards the
|
|
23
|
-
# persistent case and the score is disclosed as an upper bound
|
|
23
|
+
# persistent case and the score is disclosed as an upper bound.
|
|
24
|
+
#
|
|
25
|
+
# Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
|
|
26
|
+
# version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
|
|
27
|
+
# removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
|
|
24
28
|
module ExternalBackend
|
|
25
29
|
# Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
|
|
26
30
|
# the full suite). The per-mutant timeout is derived from how long this took.
|
|
@@ -29,6 +33,24 @@ module Mutineer
|
|
|
29
33
|
# this backend waits on an external process TREE, not an in-process fork.
|
|
30
34
|
POLL = 0.02
|
|
31
35
|
|
|
36
|
+
# PATH entries that pin a concrete Ruby under a version manager (ahead of
|
|
37
|
+
# shims). Optional trailing slash; normal bins are left alone.
|
|
38
|
+
VERSION_BIN_PATH = %r{
|
|
39
|
+
(?:
|
|
40
|
+
/\.rbenv/versions/[^/]+/bin
|
|
41
|
+
|/\.asdf/installs/ruby/[^/]+/bin
|
|
42
|
+
|/\.rubies/[^/]+/bin
|
|
43
|
+
|/rubies/[^/]+/bin
|
|
44
|
+
)/?\z
|
|
45
|
+
}x
|
|
46
|
+
|
|
47
|
+
# Env keys Process.spawn must unset (nil value) so Mutineer's Ruby/bundler
|
|
48
|
+
# context does not leak. Spawn merges env onto the parent; omitting a key
|
|
49
|
+
# does NOT clear it.
|
|
50
|
+
CLEAR_ENV_KEYS = %w[
|
|
51
|
+
BUNDLER_VERSION RBENV_VERSION ASDF_RUBY_VERSION RBENV_DIR
|
|
52
|
+
].freeze
|
|
53
|
+
|
|
32
54
|
# Turn a command template into an argv array (no shell → no eval, no
|
|
33
55
|
# injection). The `%{files}` token expands IN PLACE to N separate argv
|
|
34
56
|
# elements — one per path, unescaped — so a path containing a space stays a
|
|
@@ -41,10 +63,63 @@ module Mutineer
|
|
|
41
63
|
Shellwords.split(command).flat_map { |tok| tok == "%{files}" ? files : [tok] }
|
|
42
64
|
end
|
|
43
65
|
|
|
66
|
+
# Environment delta for the test-command child. Process.spawn merges this
|
|
67
|
+
# onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
|
|
68
|
+
# injection and version-manager PATH pins so shims / `.ruby-version` can win.
|
|
69
|
+
# Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
|
|
70
|
+
# still reaches the suite.
|
|
71
|
+
#
|
|
72
|
+
# @api private
|
|
73
|
+
# @return [Hash{String => String, nil}] env for Process.spawn.
|
|
74
|
+
def self.child_env
|
|
75
|
+
env = {}
|
|
76
|
+
cleared_pin = false
|
|
77
|
+
ENV.each_key do |k|
|
|
78
|
+
next unless clear_env_key?(k)
|
|
79
|
+
|
|
80
|
+
env[k] = nil
|
|
81
|
+
cleared_pin = true
|
|
82
|
+
end
|
|
83
|
+
path = ENV["PATH"].to_s.split(File::PATH_SEPARATOR)
|
|
84
|
+
kept = path.reject { |p| p.match?(VERSION_BIN_PATH) }
|
|
85
|
+
scrubbed_bin = kept.size != path.size
|
|
86
|
+
# Only reorder PATH when we scrubbed a pin; otherwise leave the user's PATH alone.
|
|
87
|
+
path = if cleared_pin || scrubbed_bin
|
|
88
|
+
(version_manager_shim_dirs + kept).uniq
|
|
89
|
+
else
|
|
90
|
+
kept
|
|
91
|
+
end
|
|
92
|
+
env["PATH"] = path.join(File::PATH_SEPARATOR)
|
|
93
|
+
env
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# True when the key must be unset in the child (bundler/gem/version-manager).
|
|
97
|
+
#
|
|
98
|
+
# @api private
|
|
99
|
+
# @param key [String] environment variable name.
|
|
100
|
+
# @return [Boolean]
|
|
101
|
+
def self.clear_env_key?(key)
|
|
102
|
+
key.start_with?("BUNDLE_", "RUBY", "GEM_") || CLEAR_ENV_KEYS.include?(key)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Existing shim directories for rbenv / asdf (chruby uses PATH without shims).
|
|
106
|
+
#
|
|
107
|
+
# @api private
|
|
108
|
+
# @return [Array<String>] absolute shim paths that exist.
|
|
109
|
+
def self.version_manager_shim_dirs
|
|
110
|
+
home = ENV["HOME"]
|
|
111
|
+
return [] if home.nil? || home.empty?
|
|
112
|
+
|
|
113
|
+
[
|
|
114
|
+
File.join(home, ".rbenv", "shims"),
|
|
115
|
+
File.join(home, ".asdf", "shims")
|
|
116
|
+
].select { |d| File.directory?(d) }
|
|
117
|
+
end
|
|
118
|
+
|
|
44
119
|
# Runs the command for ONE mutant against whatever is currently on disk (the
|
|
45
120
|
# caller has already swapped the mutant in via FileSwap). Maps the outcome to a
|
|
46
|
-
# Result.
|
|
47
|
-
#
|
|
121
|
+
# Result. Uses {child_env} so version-manager PATH pins from Mutineer's Ruby do
|
|
122
|
+
# not force the suite onto the wrong interpreter.
|
|
48
123
|
#
|
|
49
124
|
# @param command [String] the --test-command template.
|
|
50
125
|
# @param files [Array<String>] test file paths.
|
|
@@ -93,10 +168,42 @@ module Mutineer
|
|
|
93
168
|
else "exited #{code}"
|
|
94
169
|
end
|
|
95
170
|
detail = output.empty? ? "" : "\n--- last output ---\n#{tail(output)}"
|
|
171
|
+
hint = ruby_version_mismatch_hint(output)
|
|
96
172
|
raise SmokeCheckError,
|
|
97
|
-
"the test command #{reason} against the UNMUTATED source —
|
|
98
|
-
"
|
|
99
|
-
|
|
173
|
+
"the test command #{reason} against the UNMUTATED source — " \
|
|
174
|
+
"#{hint || generic_env_hint}.#{detail}"
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Generic smoke-failure framing when no Ruby-version mismatch is detected.
|
|
178
|
+
#
|
|
179
|
+
# @api private
|
|
180
|
+
# @return [String]
|
|
181
|
+
def self.generic_env_hint
|
|
182
|
+
"the unmutated suite is not green (failing tests, or a broken environment: " \
|
|
183
|
+
"DB, RAILS_ENV, migrations) — fix that before scoring"
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Targeted hint when Bundler reports a RubyVersionMismatch (common under
|
|
187
|
+
# rbenv/asdf/chruby when Mutineer's version bin sits ahead of shims on PATH).
|
|
188
|
+
#
|
|
189
|
+
# @api private
|
|
190
|
+
# @param output [String] captured child stdout+stderr.
|
|
191
|
+
# @return [String, nil] hint text, or nil when not a version mismatch.
|
|
192
|
+
def self.ruby_version_mismatch_hint(output)
|
|
193
|
+
return nil if output.nil? || output.empty?
|
|
194
|
+
return nil unless output.match?(/RubyVersionMismatch|Your Ruby version is .* but your Gemfile specified/i)
|
|
195
|
+
|
|
196
|
+
ran = output[/Your Ruby version is ([0-9.]+)/i, 1]
|
|
197
|
+
want = output[/Gemfile specified ([0-9.]+)/i, 1]
|
|
198
|
+
parts = +"Detected a Ruby version mismatch"
|
|
199
|
+
parts << ": the test command ran under #{ran}" if ran
|
|
200
|
+
parts << " but the app expects #{want}" if want
|
|
201
|
+
parts << ". Mutineer already scrubbed version-manager bins and RBENV_VERSION/" \
|
|
202
|
+
"ASDF_RUBY_VERSION from the child env; the suite still picked the wrong " \
|
|
203
|
+
"Ruby. Use a wrapper that re-selects the app Ruby and ensure " \
|
|
204
|
+
".ruby-version / .tool-versions is present " \
|
|
205
|
+
"(see README: Apps on Ruby < 3.4 → Under a version manager)"
|
|
206
|
+
parts
|
|
100
207
|
end
|
|
101
208
|
|
|
102
209
|
# Spawns the command to a captured combined-output tempfile, enforces a
|
|
@@ -122,7 +229,8 @@ module Mutineer
|
|
|
122
229
|
# explicit [program, argv0] form guarantees the no-shell exec path even for a
|
|
123
230
|
# degenerate single-element argv (Process.spawn(*argv) would route a lone
|
|
124
231
|
# metachar-bearing string through /bin/sh, breaking the argv-only invariant).
|
|
125
|
-
pid = Process.spawn([argv.first, argv.first], *argv[1..],
|
|
232
|
+
pid = Process.spawn(child_env, [argv.first, argv.first], *argv[1..],
|
|
233
|
+
out: out, err: %i[child out], pgroup: true)
|
|
126
234
|
kind, code = wait_with_timeout(pid, timeout)
|
|
127
235
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
128
236
|
out.rewind
|
data/lib/mutineer/file_swap.rb
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Mutineer
|
|
4
|
-
# Raised when a source file's backup already exists as FileSwap.with begins
|
|
5
|
-
#
|
|
4
|
+
# Raised when a source file's backup already exists as FileSwap.with begins.
|
|
5
|
+
# A second mutineer run is racing on the same file (the backup path is shared
|
|
6
6
|
# and unlocked). Aborting beats silently leaving the tree mutated.
|
|
7
7
|
class ConcurrentRunError < StandardError
|
|
8
8
|
def initialize(backup)
|
|
@@ -11,11 +11,11 @@ module Mutineer
|
|
|
11
11
|
end
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
#
|
|
14
|
+
# Apply one whole-file mutant to the REAL source path for the external
|
|
15
15
|
# (`--test-command`) backend, and guarantee the original is restored on every
|
|
16
|
-
# exit path. A separate `bundle exec` subprocess has its own VM and cannot see
|
|
17
|
-
# in-process `load`, so the mutant must live on disk while its suite runs
|
|
18
|
-
# makes leaving the file mutated the one genuinely dangerous failure mode.
|
|
16
|
+
# exit path. A separate `bundle exec` subprocess has its own VM and cannot see
|
|
17
|
+
# an in-process `load`, so the mutant must live on disk while its suite runs,
|
|
18
|
+
# which makes leaving the file mutated the one genuinely dangerous failure mode.
|
|
19
19
|
#
|
|
20
20
|
# Defense in depth, mirroring the tempfile-orphan discipline
|
|
21
21
|
# (`Runner.sweep_orphans`, `isolation.rb` tempfiles):
|
|
@@ -39,12 +39,13 @@ module Mutineer
|
|
|
39
39
|
# @return [Object] the block's return value.
|
|
40
40
|
def self.with(source_file, mutated)
|
|
41
41
|
backup = source_file + BACKUP_SUFFIX
|
|
42
|
-
# A backup already on disk means either a prior hard-killed run
|
|
43
|
-
# should have healed it at startup) or a SECOND mutineer
|
|
44
|
-
# same file. The backup path is shared and unlocked, so
|
|
45
|
-
# us capture the other run's mutant AS the "original"
|
|
46
|
-
# the tree. Refuse loudly rather than silently
|
|
47
|
-
# `created` is set, so the ensure below never
|
|
42
|
+
# A backup already on disk means either a prior hard-killed run
|
|
43
|
+
# (restore_orphans should have healed it at startup) or a SECOND mutineer
|
|
44
|
+
# run racing us on the same file. The backup path is shared and unlocked, so
|
|
45
|
+
# proceeding would let us capture the other run's mutant AS the "original"
|
|
46
|
+
# and permanently mutate the tree. Refuse loudly rather than silently
|
|
47
|
+
# corrupt, and do it BEFORE `created` is set, so the ensure below never
|
|
48
|
+
# touches a backup we don't own.
|
|
48
49
|
raise ConcurrentRunError, backup if File.exist?(backup)
|
|
49
50
|
|
|
50
51
|
original = File.binread(source_file)
|
|
@@ -74,11 +75,11 @@ module Mutineer
|
|
|
74
75
|
backup_bytes = File.binread(backup)
|
|
75
76
|
if !File.exist?(source_file)
|
|
76
77
|
# A real user file that merely ends in our suffix, with no sibling to
|
|
77
|
-
# restore
|
|
78
|
+
# restore. Leave it untouched (never create a file from it).
|
|
78
79
|
next
|
|
79
80
|
elsif File.binread(source_file) == backup_bytes
|
|
80
|
-
# Redundant backup (e.g. a crash between restore and unlink): nothing
|
|
81
|
-
# heal, just clear the orphan so the next run
|
|
81
|
+
# Redundant backup (e.g. a crash between restore and unlink): nothing
|
|
82
|
+
# to heal, just clear the orphan so the next run does not see a false race.
|
|
82
83
|
File.unlink(backup)
|
|
83
84
|
else
|
|
84
85
|
File.binwrite(source_file, backup_bytes)
|