evilution 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +28 -0
- data/.rubocop_todo.yml +1 -0
- data/CHANGELOG.md +28 -0
- data/README.md +8 -8
- data/docs/integrations.md +141 -0
- data/docs/isolation.md +15 -0
- data/lib/evilution/baseline.rb +11 -4
- data/lib/evilution/cli/parser/options_builder.rb +1 -1
- data/lib/evilution/config/validators/integration.rb +5 -1
- data/lib/evilution/config.rb +1 -1
- data/lib/evilution/integration/loading/mutation_applier.rb +1 -2
- data/lib/evilution/integration/loading/source_evaluator.rb +1 -2
- data/lib/evilution/integration/loading/test_load_path.rb +76 -0
- data/lib/evilution/integration/minitest.rb +6 -3
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -1
- data/lib/evilution/integration/rspec/framework_loader.rb +5 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +72 -0
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +45 -0
- data/lib/evilution/integration/rspec/state_guard.rb +3 -1
- data/lib/evilution/integration/test_unit/dispatcher.rb +26 -0
- data/lib/evilution/integration/test_unit/framework_loader.rb +33 -0
- data/lib/evilution/integration/test_unit/result_builder.rb +53 -0
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +26 -0
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +48 -0
- data/lib/evilution/integration/test_unit.rb +132 -0
- data/lib/evilution/integration/test_unit_crash_detector.rb +61 -0
- data/lib/evilution/isolation/fork.rb +45 -3
- data/lib/evilution/mcp/info_tool.rb +2 -2
- data/lib/evilution/mcp/mutate_tool.rb +3 -2
- data/lib/evilution/parallel/work_queue/dispatcher.rb +94 -22
- data/lib/evilution/parallel/work_queue/worker.rb +49 -3
- data/lib/evilution/parallel/work_queue/worker_registry.rb +47 -0
- data/lib/evilution/parallel/work_queue.rb +8 -0
- data/lib/evilution/reporter/cli/line_formatters/unresolved_rate_warning.rb +50 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +2 -0
- data/lib/evilution/runner/baseline_runner.rb +3 -1
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +28 -1
- data/lib/evilution/runner.rb +12 -1
- data/lib/evilution/spec_resolver.rb +81 -9
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +11 -0
- data/lib/tasks/stress.rake +15 -0
- data/script/run_self_baseline +2 -2
- metadata +16 -2
|
@@ -38,24 +38,47 @@ class Evilution::Parallel::WorkQueue::Dispatcher
|
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
# Each worker carries its own deadline (set when it goes busy, refreshed on
|
|
42
|
+
# every result). The select blocks only until the nearest worker deadline,
|
|
43
|
+
# so a single stuck worker is reaped in isolation -- its in-flight item gets
|
|
44
|
+
# the WorkQueue::TIMED_OUT sentinel and the worker is recycled -- instead of
|
|
45
|
+
# the old pool-wide watchdog that SIGKILLed every worker and aborted the run.
|
|
41
46
|
def collect
|
|
42
47
|
io_to_worker = @workers.to_h { |w| [w.res_io, w] }
|
|
43
48
|
result_ios = io_to_worker.keys
|
|
44
49
|
|
|
45
50
|
while @state.in_flight.positive?
|
|
46
|
-
readable, = IO.select(result_ios, nil, nil,
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
break
|
|
50
|
-
end
|
|
51
|
+
readable, = IO.select(result_ios, nil, nil, select_timeout)
|
|
52
|
+
reap_timed_out(io_to_worker, result_ios)
|
|
53
|
+
next if readable.nil?
|
|
51
54
|
|
|
52
|
-
readable.each
|
|
55
|
+
readable.each do |io|
|
|
56
|
+
process_readable(io, io_to_worker, result_ios) if result_ios.include?(io)
|
|
57
|
+
end
|
|
53
58
|
end
|
|
54
59
|
end
|
|
55
60
|
|
|
56
|
-
def
|
|
57
|
-
|
|
58
|
-
|
|
61
|
+
def select_timeout
|
|
62
|
+
return @item_timeout unless @item_timeout
|
|
63
|
+
|
|
64
|
+
deadlines = @workers.filter_map(&:deadline)
|
|
65
|
+
return @item_timeout if deadlines.empty?
|
|
66
|
+
|
|
67
|
+
[deadlines.min - monotonic, 0].max
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def reap_timed_out(io_to_worker, result_ios)
|
|
71
|
+
return unless @item_timeout
|
|
72
|
+
|
|
73
|
+
now = monotonic
|
|
74
|
+
stuck = @workers.select { |w| w.deadline && w.deadline <= now && w.pending.positive? }
|
|
75
|
+
stuck.each { |w| time_out_worker(w, io_to_worker, result_ios) }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def time_out_worker(worker, io_to_worker, result_ios)
|
|
79
|
+
worker.kill
|
|
80
|
+
mark_unfinished(worker, Evilution::Parallel::WorkQueue::TIMED_OUT)
|
|
81
|
+
retire_or_replace(worker, io_to_worker, result_ios)
|
|
59
82
|
end
|
|
60
83
|
|
|
61
84
|
def process_readable(io, io_to_worker, result_ios)
|
|
@@ -65,7 +88,7 @@ class Evilution::Parallel::WorkQueue::Dispatcher
|
|
|
65
88
|
|
|
66
89
|
def handle(worker, io_to_worker, result_ios)
|
|
67
90
|
message = worker.read_result
|
|
68
|
-
return handle_dead(worker) if message.nil?
|
|
91
|
+
return handle_dead(worker, io_to_worker, result_ios) if message.nil?
|
|
69
92
|
|
|
70
93
|
record(message, worker)
|
|
71
94
|
return false if recycle_and_dispatch(worker, io_to_worker, result_ios)
|
|
@@ -82,13 +105,24 @@ class Evilution::Parallel::WorkQueue::Dispatcher
|
|
|
82
105
|
@state.in_flight -= 1
|
|
83
106
|
worker.pending -= 1
|
|
84
107
|
worker.items_completed += 1
|
|
108
|
+
worker.in_flight_indices.delete(index)
|
|
109
|
+
worker.deadline = next_deadline(worker)
|
|
85
110
|
end
|
|
86
111
|
|
|
87
|
-
|
|
88
|
-
|
|
112
|
+
# A worker that exited without replying loses only its in-flight item(s)
|
|
113
|
+
# (marked :died) and is recycled; the run continues rather than aborting.
|
|
114
|
+
def handle_dead(worker, io_to_worker, result_ios)
|
|
115
|
+
mark_unfinished(worker, Evilution::Parallel::WorkQueue::DIED)
|
|
116
|
+
retire_or_replace(worker, io_to_worker, result_ios)
|
|
117
|
+
false
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def mark_unfinished(worker, sentinel)
|
|
121
|
+
worker.in_flight_indices.each { |index| @state.results[index] = sentinel }
|
|
89
122
|
@state.in_flight -= worker.pending
|
|
90
123
|
worker.pending = 0
|
|
91
|
-
|
|
124
|
+
worker.in_flight_indices.clear
|
|
125
|
+
worker.deadline = nil
|
|
92
126
|
end
|
|
93
127
|
|
|
94
128
|
def draining_for_recycle?(worker)
|
|
@@ -113,28 +147,66 @@ class Evilution::Parallel::WorkQueue::Dispatcher
|
|
|
113
147
|
end
|
|
114
148
|
|
|
115
149
|
def recycle(old_worker, io_to_worker, result_ios)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
@retired << old_worker.retire
|
|
119
|
-
|
|
150
|
+
index = @workers.index(old_worker)
|
|
151
|
+
detach(old_worker, io_to_worker, result_ios)
|
|
120
152
|
new_worker = @recycle_factory.call(old_worker)
|
|
121
|
-
@workers[
|
|
122
|
-
|
|
123
|
-
result_ios << new_worker.res_io
|
|
153
|
+
@workers[index] = new_worker
|
|
154
|
+
attach(new_worker, io_to_worker, result_ios)
|
|
124
155
|
new_worker
|
|
125
156
|
end
|
|
126
157
|
|
|
158
|
+
# Shared failure-path recovery: retire the worker, and as long as work
|
|
159
|
+
# remains spin up a replacement to keep the pool full and hand it the next
|
|
160
|
+
# item. When the queue is already drained, just drop the worker.
|
|
161
|
+
def retire_or_replace(worker, io_to_worker, result_ios)
|
|
162
|
+
index = @workers.index(worker)
|
|
163
|
+
detach(worker, io_to_worker, result_ios)
|
|
164
|
+
|
|
165
|
+
if more_to_send? && @state.first_error.nil?
|
|
166
|
+
new_worker = @recycle_factory.call(worker)
|
|
167
|
+
@workers[index] = new_worker
|
|
168
|
+
attach(new_worker, io_to_worker, result_ios)
|
|
169
|
+
send_item(new_worker)
|
|
170
|
+
else
|
|
171
|
+
@workers.delete_at(index)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def detach(worker, io_to_worker, result_ios)
|
|
176
|
+
io_to_worker.delete(worker.res_io)
|
|
177
|
+
result_ios.delete(worker.res_io)
|
|
178
|
+
@retired << worker.retire
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def attach(worker, io_to_worker, result_ios)
|
|
182
|
+
io_to_worker[worker.res_io] = worker
|
|
183
|
+
result_ios << worker.res_io
|
|
184
|
+
end
|
|
185
|
+
|
|
127
186
|
def send_item(worker)
|
|
128
187
|
worker.send_item(@state.next_index, @items[@state.next_index])
|
|
129
188
|
@state.next_index += 1
|
|
130
189
|
@state.in_flight += 1
|
|
190
|
+
start_deadline(worker)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def start_deadline(worker)
|
|
194
|
+
return unless @item_timeout
|
|
195
|
+
|
|
196
|
+
worker.deadline ||= monotonic + @item_timeout
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def next_deadline(worker)
|
|
200
|
+
return nil unless @item_timeout && worker.pending.positive?
|
|
201
|
+
|
|
202
|
+
monotonic + @item_timeout
|
|
131
203
|
end
|
|
132
204
|
|
|
133
205
|
def more_to_send?
|
|
134
206
|
@state.next_index < @items.length
|
|
135
207
|
end
|
|
136
208
|
|
|
137
|
-
def
|
|
138
|
-
|
|
209
|
+
def monotonic
|
|
210
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
139
211
|
end
|
|
140
212
|
end
|
|
@@ -4,12 +4,13 @@ require_relative "../work_queue"
|
|
|
4
4
|
require_relative "../../child_output"
|
|
5
5
|
require_relative "channel"
|
|
6
6
|
require_relative "channel/frame"
|
|
7
|
+
require_relative "worker_registry"
|
|
7
8
|
|
|
8
9
|
class Evilution::Parallel::WorkQueue::Worker
|
|
9
10
|
Timing = Data.define(:busy, :wall)
|
|
10
11
|
|
|
11
|
-
attr_reader :pid, :worker_index
|
|
12
|
-
attr_accessor :items_completed, :pending, :busy_time, :wall_time
|
|
12
|
+
attr_reader :pid, :worker_index, :in_flight_indices
|
|
13
|
+
attr_accessor :items_completed, :pending, :busy_time, :wall_time, :deadline
|
|
13
14
|
|
|
14
15
|
def self.spawn(worker_index:, hooks:, &block)
|
|
15
16
|
cmd_read, cmd_write = IO.pipe
|
|
@@ -26,7 +27,36 @@ class Evilution::Parallel::WorkQueue::Worker
|
|
|
26
27
|
|
|
27
28
|
cmd_read.close
|
|
28
29
|
res_write.close
|
|
29
|
-
|
|
30
|
+
# Register BEFORE isolating so the trap can never observe a worker that is
|
|
31
|
+
# already its own group leader yet missing from the registry (EV-jwao race,
|
|
32
|
+
# GH #1333 review): the spawn runs on the same main thread the trap
|
|
33
|
+
# interrupts, so a signal arriving between setpgid and register would
|
|
34
|
+
# otherwise leak a leader the trap cannot reach. Ordering register first
|
|
35
|
+
# leaves only safe windows -- pre-setpgid the child still shares the parent
|
|
36
|
+
# group and receives the terminal signal directly; once it is its own
|
|
37
|
+
# leader the registry already lists it. Registering unconditionally is safe
|
|
38
|
+
# because signal_all's kill(-pid) is a no-op (Errno::ESRCH) for a pid that
|
|
39
|
+
# never became a group leader (setpgid failed).
|
|
40
|
+
Evilution::Parallel::WorkQueue::WorkerRegistry.register(pid)
|
|
41
|
+
isolate_process_group(pid)
|
|
42
|
+
new(pid:, cmd_write:, res_read:, worker_index:)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# EV-cnx8 / GH #1324: make the worker its own process-group leader so #kill
|
|
46
|
+
# can signal the whole subtree. A mutation's spec may fork a grandchild that
|
|
47
|
+
# blocks (e.g. ConditionVariable#wait); when the dispatcher SIGKILLs a stuck
|
|
48
|
+
# worker, that grandchild must die with it rather than orphan to init holding
|
|
49
|
+
# memory/fds/connections. Done parent-side (before the child forks anything)
|
|
50
|
+
# so a failure is visible here instead of being swallowed in the child.
|
|
51
|
+
def self.isolate_process_group(pid)
|
|
52
|
+
Process.setpgid(pid, pid)
|
|
53
|
+
rescue Errno::EACCES, Errno::ESRCH
|
|
54
|
+
# EACCES: child already exec'd/changed group; ESRCH: child already exited.
|
|
55
|
+
# Both are benign -- reaping handles the child either way.
|
|
56
|
+
nil
|
|
57
|
+
rescue SystemCallError => e
|
|
58
|
+
warn "evilution: could not isolate worker #{pid} into its own process " \
|
|
59
|
+
"group (#{e.class}: #{e.message}); grandchildren may survive a kill."
|
|
30
60
|
end
|
|
31
61
|
|
|
32
62
|
# EV-kdns / GH #817: translate 0-based worker slot to parallel_tests'
|
|
@@ -46,6 +76,8 @@ class Evilution::Parallel::WorkQueue::Worker
|
|
|
46
76
|
@pending = 0
|
|
47
77
|
@busy_time = 0.0
|
|
48
78
|
@wall_time = 0.0
|
|
79
|
+
@in_flight_indices = []
|
|
80
|
+
@deadline = nil
|
|
49
81
|
end
|
|
50
82
|
|
|
51
83
|
def res_io
|
|
@@ -55,6 +87,7 @@ class Evilution::Parallel::WorkQueue::Worker
|
|
|
55
87
|
def send_item(index, item)
|
|
56
88
|
Evilution::Parallel::WorkQueue::Channel.write(@cmd_write, [index, item])
|
|
57
89
|
@pending += 1
|
|
90
|
+
@in_flight_indices << index
|
|
58
91
|
end
|
|
59
92
|
|
|
60
93
|
def read_result
|
|
@@ -67,7 +100,16 @@ class Evilution::Parallel::WorkQueue::Worker
|
|
|
67
100
|
nil
|
|
68
101
|
end
|
|
69
102
|
|
|
103
|
+
# SIGKILL the worker's whole process group (negative pid), reaping any
|
|
104
|
+
# grandchildren it forked. Falls back to the single pid if the group is gone
|
|
105
|
+
# -- already reaped, or setpgid did not take in the child.
|
|
70
106
|
def kill
|
|
107
|
+
Process.kill("KILL", -@pid)
|
|
108
|
+
rescue Errno::ESRCH
|
|
109
|
+
kill_pid
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def kill_pid
|
|
71
113
|
Process.kill("KILL", @pid)
|
|
72
114
|
rescue Errno::ESRCH
|
|
73
115
|
nil
|
|
@@ -82,6 +124,10 @@ class Evilution::Parallel::WorkQueue::Worker
|
|
|
82
124
|
Process.wait(@pid)
|
|
83
125
|
rescue Errno::ECHILD
|
|
84
126
|
nil
|
|
127
|
+
ensure
|
|
128
|
+
# Drop the pgid once the leader is reaped so the trap never signals a group
|
|
129
|
+
# whose pid the OS may have recycled. No-op if it was never registered.
|
|
130
|
+
Evilution::Parallel::WorkQueue::WorkerRegistry.unregister(@pid)
|
|
85
131
|
end
|
|
86
132
|
|
|
87
133
|
def retire
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../work_queue"
|
|
4
|
+
|
|
5
|
+
# Process-global registry of live worker process-group ids (pgids).
|
|
6
|
+
#
|
|
7
|
+
# EV-jwao / GH #1332: EV-cnx8 made each Worker its own process-group leader so a
|
|
8
|
+
# stuck worker's whole subtree can be group-killed. Side effect: a terminal
|
|
9
|
+
# Ctrl-C delivers SIGINT only to the parent's foreground group, so workers (now
|
|
10
|
+
# in their own groups) no longer receive it -- and the parent's fatal-signal
|
|
11
|
+
# death skips work_queue#map's `ensure cleanup_workers`, leaking any worker that
|
|
12
|
+
# was actively running a (possibly blocking) mutation at interrupt time.
|
|
13
|
+
#
|
|
14
|
+
# Runner#install_signal_handler reads this registry from inside the trap and
|
|
15
|
+
# forwards INT/TERM to each worker group before re-raising to DEFAULT.
|
|
16
|
+
#
|
|
17
|
+
# Signal-safety: under MRI a trap handler runs on the main thread between VM
|
|
18
|
+
# instructions, so it must not acquire a Mutex (the main thread may hold it ->
|
|
19
|
+
# deadlock). register/unregister therefore swap @pgids for a freshly built
|
|
20
|
+
# frozen array via a single atomic reference assignment (copy-on-write). The
|
|
21
|
+
# trap reads the current reference once and iterates that complete, immutable
|
|
22
|
+
# snapshot -- no torn reads, no lock.
|
|
23
|
+
module Evilution::Parallel::WorkQueue::WorkerRegistry
|
|
24
|
+
@pgids = [].freeze
|
|
25
|
+
|
|
26
|
+
class << self
|
|
27
|
+
# Frozen snapshot. Safe to read from a signal handler.
|
|
28
|
+
attr_reader :pgids
|
|
29
|
+
|
|
30
|
+
def register(pgid)
|
|
31
|
+
@pgids = (@pgids + [pgid]).freeze
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def unregister(pgid)
|
|
35
|
+
@pgids = @pgids.reject { |existing| existing == pgid }.freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def signal_all(sig)
|
|
39
|
+
@pgids.each do |pgid|
|
|
40
|
+
Process.kill(sig, -pgid)
|
|
41
|
+
rescue Errno::ESRCH
|
|
42
|
+
# Group already gone (worker + subtree reaped) -- nothing to signal.
|
|
43
|
+
nil
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -9,6 +9,14 @@ class Evilution::Parallel::WorkQueue
|
|
|
9
9
|
|
|
10
10
|
TIMING_GRACE_PERIOD = 5
|
|
11
11
|
|
|
12
|
+
# Sentinel results for items whose worker never produced a value. The
|
|
13
|
+
# dispatcher writes these into the results array (instead of aborting the
|
|
14
|
+
# whole run) so a single stuck/dead worker only loses its own in-flight
|
|
15
|
+
# item(s). Mutation-aware callers translate the reason into a status.
|
|
16
|
+
Unfinished = Data.define(:reason)
|
|
17
|
+
TIMED_OUT = Unfinished.new(reason: :timeout)
|
|
18
|
+
DIED = Unfinished.new(reason: :died)
|
|
19
|
+
|
|
12
20
|
def initialize(size:, hooks: nil, prefetch: 1, item_timeout: nil, worker_max_items: nil)
|
|
13
21
|
Validators::PositiveInt.call!(:size, size)
|
|
14
22
|
Validators::PositiveInt.call!(:prefetch, prefetch)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../line_formatters"
|
|
4
|
+
|
|
5
|
+
# EV-z7f5 / GH #1325: unresolved mutations are excluded from
|
|
6
|
+
# `score_denominator`, so a run whose specs could not be auto-resolved
|
|
7
|
+
# collapses to a bare "Score: 0.00% (0/0)" that reads like a genuine
|
|
8
|
+
# mutation-quality failure. This formatter surfaces a loud, actionable
|
|
9
|
+
# warning when the unresolved rate is high — and a distinct message when
|
|
10
|
+
# the denominator hit zero (nothing was measured at all) — so the user
|
|
11
|
+
# knows to pass --spec instead of trusting the 0.0.
|
|
12
|
+
#
|
|
13
|
+
# Sibling of ErrorRateWarning (EV-nrgw / GH #1168).
|
|
14
|
+
class Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning
|
|
15
|
+
DEFAULT_THRESHOLD = 0.25
|
|
16
|
+
HINT = "Pass --spec to point evilution at the test file(s)."
|
|
17
|
+
|
|
18
|
+
def initialize(threshold: DEFAULT_THRESHOLD)
|
|
19
|
+
@threshold = threshold
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def format(summary)
|
|
23
|
+
return nil if summary.total.zero?
|
|
24
|
+
return nil if summary.unresolved.zero?
|
|
25
|
+
|
|
26
|
+
rate = summary.unresolved.to_f / summary.total
|
|
27
|
+
return nil if rate <= @threshold
|
|
28
|
+
|
|
29
|
+
fraction = "#{summary.unresolved}/#{summary.total}"
|
|
30
|
+
pct = (rate * 100).round(1)
|
|
31
|
+
warning(summary, fraction, pct)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def warning(summary, fraction, pct)
|
|
37
|
+
if summary.unresolved == summary.total
|
|
38
|
+
"! No matching tests resolved: all #{fraction} mutations unresolved — " \
|
|
39
|
+
"no mutations were measured, so the score is not meaningful. #{HINT}"
|
|
40
|
+
elsif summary.score_denominator.zero?
|
|
41
|
+
# Denominator can also hit zero with a mix of unresolved + errors /
|
|
42
|
+
# neutral / equivalent, so do not attribute it solely to missing tests.
|
|
43
|
+
"! No mutations were measured (score not meaningful): " \
|
|
44
|
+
"#{fraction} (#{pct}%) mutations were unresolved. #{HINT}"
|
|
45
|
+
else
|
|
46
|
+
"! High unresolved rate: #{fraction} (#{pct}%) mutations had no matching " \
|
|
47
|
+
"test — score may be unreliable. #{HINT}"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -4,6 +4,7 @@ require_relative "../cli"
|
|
|
4
4
|
require_relative "line_formatters/mutations"
|
|
5
5
|
require_relative "line_formatters/score"
|
|
6
6
|
require_relative "line_formatters/error_rate_warning"
|
|
7
|
+
require_relative "line_formatters/unresolved_rate_warning"
|
|
7
8
|
require_relative "line_formatters/duration"
|
|
8
9
|
require_relative "line_formatters/efficiency"
|
|
9
10
|
require_relative "line_formatters/peak_memory"
|
|
@@ -13,6 +14,7 @@ class Evilution::Reporter::CLI::MetricsBlock
|
|
|
13
14
|
Evilution::Reporter::CLI::LineFormatters::Mutations.new,
|
|
14
15
|
Evilution::Reporter::CLI::LineFormatters::Score.new,
|
|
15
16
|
Evilution::Reporter::CLI::LineFormatters::ErrorRateWarning.new,
|
|
17
|
+
Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning.new,
|
|
16
18
|
Evilution::Reporter::CLI::LineFormatters::Duration.new,
|
|
17
19
|
Evilution::Reporter::CLI::LineFormatters::Efficiency.new,
|
|
18
20
|
Evilution::Reporter::CLI::LineFormatters::PeakMemory.new
|
|
@@ -5,6 +5,7 @@ require_relative "../baseline"
|
|
|
5
5
|
require_relative "../spec_resolver"
|
|
6
6
|
require_relative "../integration/rspec"
|
|
7
7
|
require_relative "../integration/minitest"
|
|
8
|
+
require_relative "../integration/test_unit"
|
|
8
9
|
require_relative "../example_filter"
|
|
9
10
|
require_relative "../spec_ast_cache"
|
|
10
11
|
require_relative "../source_ast_cache"
|
|
@@ -12,7 +13,8 @@ require_relative "../source_ast_cache"
|
|
|
12
13
|
unless defined?(Evilution::Runner::INTEGRATIONS)
|
|
13
14
|
Evilution::Runner::INTEGRATIONS = {
|
|
14
15
|
rspec: Evilution::Integration::RSpec,
|
|
15
|
-
minitest: Evilution::Integration::Minitest
|
|
16
|
+
minitest: Evilution::Integration::Minitest,
|
|
17
|
+
test_unit: Evilution::Integration::TestUnit
|
|
16
18
|
}.freeze
|
|
17
19
|
end
|
|
18
20
|
|
|
@@ -1,8 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../strategy"
|
|
4
|
+
require_relative "../../../parallel/work_queue"
|
|
4
5
|
|
|
5
6
|
class Evilution::Runner::MutationExecutor::Strategy::Parallel
|
|
7
|
+
# Compact result for a worker that exited unexpectedly (:error is not cached,
|
|
8
|
+
# so the 0.0 duration is never persisted).
|
|
9
|
+
DIED_COMPACT = {
|
|
10
|
+
status: :error,
|
|
11
|
+
duration: 0.0,
|
|
12
|
+
error_message: "worker process exited unexpectedly",
|
|
13
|
+
error_class: "Evilution::Error"
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
6
16
|
def initialize(cache:, isolator:, packer:, pipeline:, notifier:, pool_factory:, config:, diagnostics: nil)
|
|
7
17
|
@cache = cache
|
|
8
18
|
@isolator = isolator
|
|
@@ -71,11 +81,28 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
|
|
|
71
81
|
return [] if uncached_indices.empty?
|
|
72
82
|
|
|
73
83
|
uncached = uncached_indices.map { |i| batch[i] }
|
|
74
|
-
pool.map(uncached) do |mutation|
|
|
84
|
+
worker_results = pool.map(uncached) do |mutation|
|
|
75
85
|
test_command = ->(m) { integration.call(m) }
|
|
76
86
|
result = @isolator.call(mutation: mutation, test_command: test_command, timeout: @config.timeout)
|
|
77
87
|
@packer.compact(result)
|
|
78
88
|
end
|
|
89
|
+
worker_results.map { |r| unpack_unfinished(r) }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def unpack_unfinished(result)
|
|
93
|
+
return result unless result.is_a?(Evilution::Parallel::WorkQueue::Unfinished)
|
|
94
|
+
|
|
95
|
+
case result.reason
|
|
96
|
+
when :timeout then timeout_compact
|
|
97
|
+
when :died then DIED_COMPACT
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# A :timeout result is cacheable, so give it a realistic duration: the stuck
|
|
102
|
+
# worker exhausted the dispatcher's item_timeout (config.timeout * 2) before
|
|
103
|
+
# being killed. A 0.0 here would be cached and skew summaries/reuse.
|
|
104
|
+
def timeout_compact
|
|
105
|
+
{ status: :timeout, duration: @config.timeout * 2.0 }
|
|
79
106
|
end
|
|
80
107
|
|
|
81
108
|
def merge(batch, uncached_indices, cached_results, worker_results)
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -154,7 +154,12 @@ class Evilution::Runner
|
|
|
154
154
|
return
|
|
155
155
|
end
|
|
156
156
|
|
|
157
|
-
|
|
157
|
+
# Resolve to absolute now — fork children chdir into a per-mutation
|
|
158
|
+
# sandbox (EV-wqxu / GH #1278) before suppress_child_output runs, so a
|
|
159
|
+
# relative log_dir would reopen $stdout/$stderr at <sandbox>/<log_dir>
|
|
160
|
+
# which does not exist and the child dies with Errno::ENOENT before
|
|
161
|
+
# marshaling any result back.
|
|
162
|
+
dir = File.expand_path(config.quiet_children_dir)
|
|
158
163
|
begin
|
|
159
164
|
FileUtils.rm_rf(dir)
|
|
160
165
|
FileUtils.mkdir_p(dir)
|
|
@@ -173,6 +178,11 @@ class Evilution::Runner
|
|
|
173
178
|
def install_signal_handler(sig)
|
|
174
179
|
prev_handler = Signal.trap(sig) do
|
|
175
180
|
Evilution::TempDirTracker.cleanup_all
|
|
181
|
+
# EV-jwao / GH #1332: workers are their own process-group leaders, so a
|
|
182
|
+
# terminal Ctrl-C reaches only the parent's group. Forward to each worker
|
|
183
|
+
# group here -- the parent's fatal-signal death skips work_queue#map's
|
|
184
|
+
# `ensure cleanup_workers`, so this trap is the reliable forwarding hook.
|
|
185
|
+
Evilution::Parallel::WorkQueue::WorkerRegistry.signal_all(sig)
|
|
176
186
|
|
|
177
187
|
case prev_handler
|
|
178
188
|
when Proc, Method
|
|
@@ -221,6 +231,7 @@ require_relative "result/summary"
|
|
|
221
231
|
require_relative "baseline"
|
|
222
232
|
require_relative "cache"
|
|
223
233
|
require_relative "parallel/pool"
|
|
234
|
+
require_relative "parallel/work_queue/worker_registry"
|
|
224
235
|
require_relative "session/store"
|
|
225
236
|
require_relative "temp_dir_tracker"
|
|
226
237
|
require_relative "rails_detector"
|
|
@@ -3,6 +3,11 @@
|
|
|
3
3
|
class Evilution::SpecResolver
|
|
4
4
|
STRIPPABLE_PREFIXES = %w[lib/ app/].freeze
|
|
5
5
|
CONTROLLER_PREFIX = "controllers/"
|
|
6
|
+
# Conventional test subdirectories appended to @test_dir. Real-world gems
|
|
7
|
+
# frequently park specs under spec/unit or spec/lib (test/unit, test/lib)
|
|
8
|
+
# rather than mirroring the lib/ tree 1:1 (EV-z7f5 / GH #1325).
|
|
9
|
+
CONVENTIONAL_SUBDIRS = %w[unit lib].freeze
|
|
10
|
+
MINITEST_SUFFIX = "_test.rb"
|
|
6
11
|
|
|
7
12
|
def initialize(test_dir: "spec", test_suffix: "_spec.rb", request_dir: "requests")
|
|
8
13
|
@test_dir = test_dir
|
|
@@ -23,8 +28,38 @@ class Evilution::SpecResolver
|
|
|
23
28
|
Array(source_paths).filter_map { |path| call(path) }.uniq
|
|
24
29
|
end
|
|
25
30
|
|
|
31
|
+
# Best-guess candidate for an unresolved source, found by basename glob
|
|
32
|
+
# rather than the deterministic path mirroring used by #call. Used only to
|
|
33
|
+
# enrich the "no matching test" hint (EV-z7f5 / GH #1325) — never to pick a
|
|
34
|
+
# test to run — so a fuzzy substring match is acceptable here. Returns the
|
|
35
|
+
# shallowest match, or nil when nothing resembles the basename.
|
|
36
|
+
def suggest(source_path)
|
|
37
|
+
return nil if source_path.nil? || source_path.empty?
|
|
38
|
+
|
|
39
|
+
stem = File.basename(normalize_path(source_path), ".rb")
|
|
40
|
+
return nil if stem.empty?
|
|
41
|
+
|
|
42
|
+
suggestion_globs(stem).flat_map { |glob| glob_relative(glob) }.uniq.min_by(&:length)
|
|
43
|
+
end
|
|
44
|
+
|
|
26
45
|
private
|
|
27
46
|
|
|
47
|
+
# Glob for project-relative paths. Mirrors #call's project_relative_exists?
|
|
48
|
+
# contract: when run inside an isolated worker chdir'd into a sandbox, glob
|
|
49
|
+
# against PROJECT_ROOT so suggestions still find real project files. base:
|
|
50
|
+
# already yields paths relative to the root, matching the CWD-glob shape.
|
|
51
|
+
def glob_relative(glob)
|
|
52
|
+
return Dir.glob(glob) unless Evilution.in_isolated_worker?
|
|
53
|
+
|
|
54
|
+
Dir.glob(glob, base: Evilution::PROJECT_ROOT)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def suggestion_globs(stem)
|
|
58
|
+
globs = ["#{@test_dir}/**/*#{stem}*#{@test_suffix}"]
|
|
59
|
+
globs << "#{@test_dir}/**/test_#{stem}.rb" if @test_suffix == MINITEST_SUFFIX
|
|
60
|
+
globs
|
|
61
|
+
end
|
|
62
|
+
|
|
28
63
|
# Existence check that succeeds against the current CWD. When the caller
|
|
29
64
|
# is an isolated worker that chdir'd into a per-mutation sandbox (Evilution
|
|
30
65
|
# signals this via in_isolated_worker?), also try PROJECT_ROOT so the
|
|
@@ -52,18 +87,55 @@ class Evilution::SpecResolver
|
|
|
52
87
|
def candidate_test_paths(source_path)
|
|
53
88
|
base = source_path.sub(/\.rb\z/, @test_suffix)
|
|
54
89
|
prefix = STRIPPABLE_PREFIXES.find { |p| source_path.start_with?(p) }
|
|
90
|
+
stripped = prefix ? base.delete_prefix(prefix) : base
|
|
91
|
+
|
|
92
|
+
primary = mirror_candidates(stripped)
|
|
93
|
+
primary.unshift(controller_to_request_test(stripped)) if prefix
|
|
94
|
+
primary.compact!
|
|
55
95
|
|
|
56
|
-
|
|
57
|
-
stripped = base.delete_prefix(prefix)
|
|
58
|
-
request_test = controller_to_request_test(stripped)
|
|
59
|
-
[request_test, "#{@test_dir}/#{stripped}", "#{@test_dir}/#{base}"].compact
|
|
60
|
-
else
|
|
61
|
-
["#{@test_dir}/#{base}"]
|
|
62
|
-
end
|
|
96
|
+
fallbacks = primary.flat_map { |c| parent_fallback_candidates(c) }
|
|
63
97
|
|
|
64
|
-
|
|
98
|
+
(primary + fallbacks + prefix_convention_candidates(stripped)).uniq
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Conventional roots that may hold tests: the mirrored root plus the common
|
|
102
|
+
# spec/unit, spec/lib (test/unit, test/lib) buckets.
|
|
103
|
+
def roots
|
|
104
|
+
[@test_dir, *CONVENTIONAL_SUBDIRS.map { |d| "#{@test_dir}/#{d}" }]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Cross every conventional root with every layout variant of the stripped
|
|
108
|
+
# source path: the full mirror, the mirror with the leading gem-namespace
|
|
109
|
+
# dir dropped, and the bare basename. Full mirrors rank above dropped ones
|
|
110
|
+
# so a 1:1 layout always wins when present.
|
|
111
|
+
def mirror_candidates(stripped)
|
|
112
|
+
mirror_variants(stripped).flat_map do |variant|
|
|
113
|
+
roots.map { |root| "#{root}/#{variant}" }
|
|
114
|
+
end
|
|
115
|
+
end
|
|
65
116
|
|
|
66
|
-
|
|
117
|
+
def mirror_variants(stripped)
|
|
118
|
+
segments = stripped.split("/")
|
|
119
|
+
variants = [stripped]
|
|
120
|
+
variants << segments[1..].join("/") if segments.length > 1
|
|
121
|
+
variants << segments.last if segments.length > 2
|
|
122
|
+
variants.uniq
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Test::Unit / minitest gems frequently name files with a `test_` PREFIX
|
|
126
|
+
# (test/test_connection_pool.rb) instead of the mirrored `_test.rb` suffix.
|
|
127
|
+
# Only meaningful when resolving against the minitest suffix.
|
|
128
|
+
def prefix_convention_candidates(stripped)
|
|
129
|
+
return [] unless @test_suffix == MINITEST_SUFFIX
|
|
130
|
+
|
|
131
|
+
mirror_variants(stripped).flat_map do |variant|
|
|
132
|
+
dir, _, file = variant.rpartition("/")
|
|
133
|
+
name = file.delete_suffix(@test_suffix)
|
|
134
|
+
next [] if name.empty?
|
|
135
|
+
|
|
136
|
+
relative = dir.empty? ? "test_#{name}.rb" : "#{dir}/test_#{name}.rb"
|
|
137
|
+
roots.map { |root| "#{root}/#{relative}" }
|
|
138
|
+
end
|
|
67
139
|
end
|
|
68
140
|
|
|
69
141
|
def controller_to_request_test(stripped_path)
|
data/lib/evilution/version.rb
CHANGED
data/lib/evilution.rb
CHANGED
|
@@ -157,6 +157,17 @@ module Evilution
|
|
|
157
157
|
@in_isolated_worker = previous
|
|
158
158
|
end
|
|
159
159
|
|
|
160
|
+
# Base directory for resolving project-relative paths. An isolated worker
|
|
161
|
+
# has chdir'd into a per-mutation sandbox (EV-wqxu / GH #1278), so callers
|
|
162
|
+
# in that context must anchor against PROJECT_ROOT rather than Dir.pwd —
|
|
163
|
+
# otherwise spec files, source eval __FILE__, and $LOAD_PATH entries
|
|
164
|
+
# resolve into the sandbox and break the run. In any other context (normal
|
|
165
|
+
# use, tests that intentionally chdir into a fixture project layout, etc.)
|
|
166
|
+
# the caller's Dir.pwd remains the truth.
|
|
167
|
+
def self.project_base_dir
|
|
168
|
+
in_isolated_worker? ? PROJECT_ROOT : Dir.pwd
|
|
169
|
+
end
|
|
170
|
+
|
|
160
171
|
class Error < StandardError
|
|
161
172
|
attr_reader :file
|
|
162
173
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec/core/rake_task"
|
|
4
|
+
|
|
5
|
+
# RUN_STRESS lifts the default :stress exclusion in spec_helper. Set via a
|
|
6
|
+
# prerequisite so it runs before the RSpec task, without polluting other tasks.
|
|
7
|
+
task :stress_env do
|
|
8
|
+
ENV["RUN_STRESS"] = "1"
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
desc "Run parallel/isolation stress + load specs (tagged :stress, slow)"
|
|
12
|
+
RSpec::Core::RakeTask.new(stress: :stress_env) do |t|
|
|
13
|
+
t.pattern = "spec/evilution/parallel/stress_spec.rb"
|
|
14
|
+
t.rspec_opts = "--tag stress"
|
|
15
|
+
end
|
data/script/run_self_baseline
CHANGED
|
@@ -38,7 +38,7 @@ dirs.each do |dir|
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
log = File.join(LOG_DIR, "#{dir}.self.log")
|
|
41
|
-
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", *files]
|
|
41
|
+
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", "--isolation=fork", *files]
|
|
42
42
|
puts "==> #{dir} (#{files.length} files)"
|
|
43
43
|
pid = spawn(*cmd, out: log, err: %i[child out])
|
|
44
44
|
Process.wait(pid)
|
|
@@ -53,7 +53,7 @@ toplevel_files = Dir.glob(File.join(ROOT, "lib", "evilution", "*.rb"))
|
|
|
53
53
|
toplevel_files.reject! { |f| SKIP_FILES.include?(f) }
|
|
54
54
|
unless toplevel_files.empty?
|
|
55
55
|
log = File.join(LOG_DIR, "toplevel.self.log")
|
|
56
|
-
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", *toplevel_files]
|
|
56
|
+
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", "--isolation=fork", *toplevel_files]
|
|
57
57
|
puts "==> toplevel (#{toplevel_files.length} files)"
|
|
58
58
|
pid = spawn(*cmd, out: log, err: %i[child out])
|
|
59
59
|
Process.wait(pid)
|