evilution 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +5 -0
- data/CHANGELOG.md +16 -0
- data/README.md +1 -0
- data/lib/evilution/cli/parser/command_extractor.rb +77 -0
- data/lib/evilution/cli/parser/file_args.rb +41 -0
- data/lib/evilution/cli/parser/options_builder.rb +103 -0
- data/lib/evilution/cli/parser/stdin_reader.rb +28 -0
- data/lib/evilution/cli/parser.rb +27 -196
- data/lib/evilution/config.rb +14 -1
- data/lib/evilution/integration/base.rb +11 -57
- data/lib/evilution/integration/minitest.rb +16 -3
- data/lib/evilution/integration/rspec.rb +19 -7
- data/lib/evilution/isolation/fork.rb +1 -0
- data/lib/evilution/isolation/in_process.rb +1 -0
- data/lib/evilution/reporter/cli.rb +2 -1
- data/lib/evilution/reporter/html/assets/style.css +68 -0
- data/lib/evilution/reporter/html/baseline_keys.rb +28 -0
- data/lib/evilution/reporter/html/diff_formatter.rb +27 -0
- data/lib/evilution/reporter/html/escape.rb +12 -0
- data/lib/evilution/reporter/html/namespace.rb +11 -0
- data/lib/evilution/reporter/html/report.rb +68 -0
- data/lib/evilution/reporter/html/section.rb +21 -0
- data/lib/evilution/reporter/html/sections/baseline_comparison.rb +46 -0
- data/lib/evilution/reporter/html/sections/error_details.rb +30 -0
- data/lib/evilution/reporter/html/sections/error_entry.rb +22 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +47 -0
- data/lib/evilution/reporter/html/sections/header.rb +29 -0
- data/lib/evilution/reporter/html/sections/mutation_map.rb +32 -0
- data/lib/evilution/reporter/html/sections/summary_cards.rb +11 -0
- data/lib/evilution/reporter/html/sections/survived_details.rb +35 -0
- data/lib/evilution/reporter/html/sections/survived_entry.rb +36 -0
- data/lib/evilution/reporter/html/sections/truncation_notice.rb +17 -0
- data/lib/evilution/reporter/html/sections.rb +4 -0
- data/lib/evilution/reporter/html/stylesheet.rb +14 -0
- data/lib/evilution/reporter/html/templates/baseline_comparison.html.erb +8 -0
- data/lib/evilution/reporter/html/templates/error_details.html.erb +6 -0
- data/lib/evilution/reporter/html/templates/error_entry.html.erb +10 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +9 -0
- data/lib/evilution/reporter/html/templates/header.html.erb +4 -0
- data/lib/evilution/reporter/html/templates/mutation_map.html.erb +6 -0
- data/lib/evilution/reporter/html/templates/report.html.erb +17 -0
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +23 -0
- data/lib/evilution/reporter/html/templates/survived_details.html.erb +21 -0
- data/lib/evilution/reporter/html/templates/survived_entry.html.erb +8 -0
- data/lib/evilution/reporter/html/templates/truncation_notice.html.erb +1 -0
- data/lib/evilution/reporter/html.rb +11 -390
- data/lib/evilution/reporter/json.rb +12 -8
- data/lib/evilution/result/mutation_result.rb +5 -1
- data/lib/evilution/result/summary.rb +9 -1
- data/lib/evilution/runner/baseline_runner.rb +71 -0
- data/lib/evilution/runner/diagnostics.rb +105 -0
- data/lib/evilution/runner/isolation_resolver.rb +134 -0
- data/lib/evilution/runner/mutation_executor.rb +255 -0
- data/lib/evilution/runner/mutation_planner.rb +126 -0
- data/lib/evilution/runner/report_publisher.rb +60 -0
- data/lib/evilution/runner/subject_pipeline.rb +121 -0
- data/lib/evilution/runner.rb +57 -694
- data/lib/evilution/version.rb +1 -1
- metadata +42 -1
data/lib/evilution/runner.rb
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "config"
|
|
4
4
|
require_relative "ast/parser"
|
|
5
|
-
require_relative "ast/inheritance_scanner"
|
|
6
5
|
require_relative "memory"
|
|
7
6
|
require_relative "mutator/registry"
|
|
8
7
|
require_relative "isolation/fork"
|
|
@@ -13,7 +12,6 @@ require_relative "reporter/json"
|
|
|
13
12
|
require_relative "reporter/cli"
|
|
14
13
|
require_relative "reporter/html"
|
|
15
14
|
require_relative "reporter/suggestion"
|
|
16
|
-
require_relative "equivalent/detector"
|
|
17
15
|
require_relative "git/changed_files"
|
|
18
16
|
require_relative "result/mutation_result"
|
|
19
17
|
require_relative "result/summary"
|
|
@@ -21,23 +19,17 @@ require_relative "baseline"
|
|
|
21
19
|
require_relative "cache"
|
|
22
20
|
require_relative "parallel/pool"
|
|
23
21
|
require_relative "session/store"
|
|
24
|
-
require_relative "ast/pattern/filter"
|
|
25
22
|
require_relative "temp_dir_tracker"
|
|
26
|
-
require_relative "disable_comment"
|
|
27
|
-
require_relative "ast/sorbet_sig_detector"
|
|
28
23
|
require_relative "rails_detector"
|
|
24
|
+
require_relative "runner/subject_pipeline"
|
|
25
|
+
require_relative "runner/mutation_planner"
|
|
26
|
+
require_relative "runner/isolation_resolver"
|
|
27
|
+
require_relative "runner/baseline_runner"
|
|
28
|
+
require_relative "runner/diagnostics"
|
|
29
|
+
require_relative "runner/mutation_executor"
|
|
30
|
+
require_relative "runner/report_publisher"
|
|
29
31
|
|
|
30
32
|
class Evilution::Runner
|
|
31
|
-
INTEGRATIONS = {
|
|
32
|
-
rspec: Evilution::Integration::RSpec,
|
|
33
|
-
minitest: Evilution::Integration::Minitest
|
|
34
|
-
}.freeze
|
|
35
|
-
|
|
36
|
-
PRELOAD_CANDIDATES = [
|
|
37
|
-
File.join("spec", "rails_helper.rb"),
|
|
38
|
-
File.join("test", "test_helper.rb")
|
|
39
|
-
].freeze
|
|
40
|
-
|
|
41
33
|
attr_reader :config
|
|
42
34
|
|
|
43
35
|
def initialize(config: Evilution::Config.new, on_result: nil, hooks: nil)
|
|
@@ -47,17 +39,13 @@ class Evilution::Runner
|
|
|
47
39
|
@parser = Evilution::AST::Parser.new
|
|
48
40
|
@registry = Evilution::Mutator::Registry.default
|
|
49
41
|
@cache = config.incremental? ? Evilution::Cache.new : nil
|
|
50
|
-
@disable_detector = Evilution::DisableComment.new
|
|
51
|
-
@disabled_ranges_cache = {}
|
|
52
|
-
@sig_detector = Evilution::AST::SorbetSigDetector.new
|
|
53
|
-
@sig_ranges_cache = {}
|
|
54
42
|
end
|
|
55
43
|
|
|
56
44
|
def call
|
|
57
45
|
install_signal_handlers
|
|
58
46
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
59
47
|
|
|
60
|
-
subjects =
|
|
48
|
+
subjects = subject_pipeline.call
|
|
61
49
|
log_memory("after parse_subjects", "#{subjects.length} subjects")
|
|
62
50
|
|
|
63
51
|
perform_preload
|
|
@@ -65,22 +53,18 @@ class Evilution::Runner
|
|
|
65
53
|
|
|
66
54
|
baseline_result = run_baseline(subjects)
|
|
67
55
|
|
|
68
|
-
|
|
69
|
-
equivalent_mutations, mutations = filter_equivalent(mutations)
|
|
56
|
+
plan = mutation_planner.call(subjects)
|
|
70
57
|
release_subject_nodes(subjects)
|
|
71
58
|
clear_operator_caches
|
|
72
|
-
results, truncated = run_mutations(
|
|
73
|
-
results +=
|
|
74
|
-
m.strip_sources!
|
|
75
|
-
equivalent_result(m)
|
|
76
|
-
end
|
|
59
|
+
results, truncated = run_mutations(plan.enabled, baseline_result)
|
|
60
|
+
results += equivalent_results(plan.equivalent)
|
|
77
61
|
log_memory("after run_mutations", "#{results.length} results")
|
|
78
62
|
|
|
79
63
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
|
|
80
64
|
|
|
81
65
|
summary = Evilution::Result::Summary.new(results: results, duration: duration, truncated: truncated,
|
|
82
|
-
skipped: skipped_count,
|
|
83
|
-
disabled_mutations: disabled_mutations)
|
|
66
|
+
skipped: plan.skipped_count,
|
|
67
|
+
disabled_mutations: plan.disabled_mutations)
|
|
84
68
|
output_report(summary)
|
|
85
69
|
save_session(summary)
|
|
86
70
|
|
|
@@ -88,207 +72,39 @@ class Evilution::Runner
|
|
|
88
72
|
end
|
|
89
73
|
|
|
90
74
|
def parse_and_filter_subjects
|
|
91
|
-
|
|
92
|
-
subjects = filter_by_descendants(subjects) if descendants_target?
|
|
93
|
-
subjects = filter_by_target(subjects) if method_target?
|
|
94
|
-
subjects = filter_by_line_ranges(subjects) if config.line_ranges?
|
|
95
|
-
subjects
|
|
75
|
+
subject_pipeline.call
|
|
96
76
|
end
|
|
97
77
|
|
|
98
78
|
private
|
|
99
79
|
|
|
100
|
-
attr_reader :parser, :registry, :cache, :on_result, :hooks
|
|
101
|
-
|
|
102
|
-
def isolator
|
|
103
|
-
@isolator ||= build_isolator
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
def parse_subjects
|
|
107
|
-
files = resolve_target_files
|
|
108
|
-
files.flat_map { |file| parser.call(file) }
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
def resolve_target_files
|
|
112
|
-
@resolve_target_files ||= if source_glob_target?
|
|
113
|
-
resolve_source_glob
|
|
114
|
-
elsif !config.target_files.empty?
|
|
115
|
-
config.target_files
|
|
116
|
-
else
|
|
117
|
-
Evilution::Git::ChangedFiles.new.call
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
def source_glob_target?
|
|
122
|
-
config.target&.start_with?("source:")
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
def descendants_target?
|
|
126
|
-
config.target&.start_with?("descendants:")
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
def method_target?
|
|
130
|
-
config.target? && !source_glob_target? && !descendants_target?
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
def resolve_source_glob
|
|
134
|
-
pattern = config.target.delete_prefix("source:")
|
|
135
|
-
files = Dir.glob(pattern)
|
|
136
|
-
raise Evilution::Error, "no files found matching '#{pattern}'" if files.empty?
|
|
137
|
-
|
|
138
|
-
files.sort
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
def filter_by_descendants(subjects)
|
|
142
|
-
base_name = config.target.delete_prefix("descendants:")
|
|
143
|
-
files = resolve_target_files
|
|
144
|
-
inheritance = Evilution::AST::InheritanceScanner.call(files)
|
|
145
|
-
class_names = resolve_descendant_set(base_name, inheritance)
|
|
146
|
-
raise Evilution::Error, "no classes found matching '#{config.target}'" if class_names.empty?
|
|
147
|
-
|
|
148
|
-
subjects.select { |s| class_names.include?(s.name.split(/[#.]/).first) }
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
def resolve_descendant_set(base_name, inheritance)
|
|
152
|
-
descendants = Set.new
|
|
153
|
-
known = inheritance.key?(base_name) || inheritance.value?(base_name)
|
|
154
|
-
return descendants unless known
|
|
155
|
-
|
|
156
|
-
descendants.add(base_name)
|
|
157
|
-
changed = true
|
|
158
|
-
while changed
|
|
159
|
-
changed = false
|
|
160
|
-
inheritance.each do |child, parent|
|
|
161
|
-
next unless descendants.include?(parent)
|
|
162
|
-
next if descendants.include?(child)
|
|
163
|
-
|
|
164
|
-
descendants.add(child)
|
|
165
|
-
changed = true
|
|
166
|
-
end
|
|
167
|
-
end
|
|
168
|
-
descendants
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def filter_by_target(subjects)
|
|
172
|
-
matched = subjects.select(&target_matcher)
|
|
173
|
-
raise Evilution::Error, "no method found matching '#{config.target}'" if matched.empty?
|
|
174
|
-
|
|
175
|
-
matched
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
def target_matcher
|
|
179
|
-
target = config.target
|
|
180
|
-
if target.end_with?("*")
|
|
181
|
-
prefix = target.chomp("*")
|
|
182
|
-
->(s) { s.name.split(/[#.]/).first.start_with?(prefix) }
|
|
183
|
-
elsif target.end_with?("#", ".")
|
|
184
|
-
prefix = target
|
|
185
|
-
->(s) { s.name.start_with?(prefix) }
|
|
186
|
-
elsif target.include?("#") || target.include?(".")
|
|
187
|
-
->(s) { s.name == target }
|
|
188
|
-
else
|
|
189
|
-
->(s) { s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
|
|
190
|
-
end
|
|
191
|
-
end
|
|
192
|
-
|
|
193
|
-
def filter_by_line_ranges(subjects)
|
|
194
|
-
subjects.select do |subject|
|
|
195
|
-
range = config.line_ranges[subject.file_path]
|
|
196
|
-
next true unless range
|
|
197
|
-
|
|
198
|
-
subject_start = subject.line_number
|
|
199
|
-
subject_end = subject_start + subject.source.count("\n")
|
|
200
|
-
subject_start <= range.last && subject_end >= range.first
|
|
201
|
-
end
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
def generate_mutations(subjects)
|
|
205
|
-
filter = build_ignore_filter
|
|
206
|
-
operator_options = build_operator_options
|
|
207
|
-
mutations = subjects.flat_map do |subject|
|
|
208
|
-
registry.mutations_for(subject, filter: filter, operator_options: operator_options)
|
|
209
|
-
end
|
|
210
|
-
skipped_count = filter ? filter.skipped_count : 0
|
|
211
|
-
|
|
212
|
-
mutations, disabled = filter_disabled(mutations)
|
|
213
|
-
disabled.each(&:strip_sources!) if config.show_disabled?
|
|
214
|
-
disabled_mutations = config.show_disabled? ? disabled : []
|
|
80
|
+
attr_reader :parser, :registry, :cache, :on_result, :hooks
|
|
215
81
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
[mutations, skipped_count + disabled.length + sig_skipped, disabled_mutations]
|
|
82
|
+
def subject_pipeline
|
|
83
|
+
@subject_pipeline ||= Evilution::Runner::SubjectPipeline.new(config, parser: parser)
|
|
219
84
|
end
|
|
220
85
|
|
|
221
|
-
def
|
|
222
|
-
|
|
223
|
-
disabled = []
|
|
224
|
-
|
|
225
|
-
mutations.each do |mutation|
|
|
226
|
-
if mutation_disabled?(mutation)
|
|
227
|
-
disabled << mutation
|
|
228
|
-
else
|
|
229
|
-
enabled << mutation
|
|
230
|
-
end
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
[enabled, disabled]
|
|
86
|
+
def mutation_planner
|
|
87
|
+
@mutation_planner ||= Evilution::Runner::MutationPlanner.new(config, registry: registry)
|
|
234
88
|
end
|
|
235
89
|
|
|
236
|
-
def
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
@disabled_ranges_cache[file_path] ||= begin
|
|
243
|
-
source = File.read(file_path)
|
|
244
|
-
@disable_detector.call(source)
|
|
245
|
-
rescue SystemCallError
|
|
246
|
-
[]
|
|
247
|
-
end
|
|
248
|
-
end
|
|
249
|
-
|
|
250
|
-
def filter_sig_blocks(mutations)
|
|
251
|
-
enabled = []
|
|
252
|
-
skipped = 0
|
|
253
|
-
|
|
254
|
-
mutations.each do |mutation|
|
|
255
|
-
if mutation_in_sig_block?(mutation)
|
|
256
|
-
skipped += 1
|
|
257
|
-
else
|
|
258
|
-
enabled << mutation
|
|
259
|
-
end
|
|
260
|
-
end
|
|
261
|
-
|
|
262
|
-
[enabled, skipped]
|
|
263
|
-
end
|
|
264
|
-
|
|
265
|
-
def mutation_in_sig_block?(mutation)
|
|
266
|
-
ranges = sig_line_ranges_for(mutation.file_path)
|
|
267
|
-
ranges.any? { |range| range.cover?(mutation.line) }
|
|
268
|
-
end
|
|
269
|
-
|
|
270
|
-
def sig_line_ranges_for(file_path)
|
|
271
|
-
@sig_ranges_cache[file_path] ||= begin
|
|
272
|
-
source = File.read(file_path)
|
|
273
|
-
@sig_detector.line_ranges(source)
|
|
274
|
-
rescue SystemCallError
|
|
275
|
-
[]
|
|
276
|
-
end
|
|
90
|
+
def isolation_resolver
|
|
91
|
+
@isolation_resolver ||= Evilution::Runner::IsolationResolver.new(
|
|
92
|
+
config,
|
|
93
|
+
target_files: -> { subject_pipeline.target_files },
|
|
94
|
+
hooks: @hooks
|
|
95
|
+
)
|
|
277
96
|
end
|
|
278
97
|
|
|
279
|
-
def
|
|
280
|
-
|
|
98
|
+
def isolator
|
|
99
|
+
isolation_resolver.isolator
|
|
281
100
|
end
|
|
282
101
|
|
|
283
|
-
def
|
|
284
|
-
|
|
285
|
-
return nil if patterns.nil? || patterns.empty?
|
|
286
|
-
|
|
287
|
-
Evilution::AST::Pattern::Filter.new(patterns)
|
|
102
|
+
def rails_root_detected?
|
|
103
|
+
isolation_resolver.rails_root_detected?
|
|
288
104
|
end
|
|
289
105
|
|
|
290
|
-
def
|
|
291
|
-
|
|
106
|
+
def perform_preload
|
|
107
|
+
isolation_resolver.perform_preload
|
|
292
108
|
end
|
|
293
109
|
|
|
294
110
|
def release_subject_nodes(subjects)
|
|
@@ -299,175 +115,39 @@ class Evilution::Runner
|
|
|
299
115
|
Evilution::Mutator::Base.clear_parse_cache!
|
|
300
116
|
end
|
|
301
117
|
|
|
302
|
-
def
|
|
303
|
-
|
|
304
|
-
end
|
|
305
|
-
|
|
306
|
-
def run_baseline(subjects)
|
|
307
|
-
return nil unless config.baseline? && subjects.any?
|
|
308
|
-
|
|
309
|
-
log_baseline_start
|
|
310
|
-
integration_class = resolve_integration_class
|
|
311
|
-
baseline = Evilution::Baseline.new(timeout: config.timeout, **integration_class.baseline_options)
|
|
312
|
-
result = baseline.call(subjects)
|
|
313
|
-
log_baseline_complete(result)
|
|
314
|
-
result
|
|
315
|
-
end
|
|
316
|
-
|
|
317
|
-
def run_mutations(mutations, baseline_result = nil)
|
|
318
|
-
@progress_bar = build_progress_bar(mutations.length)
|
|
319
|
-
result = if config.jobs > 1
|
|
320
|
-
run_mutations_parallel(mutations, baseline_result)
|
|
321
|
-
else
|
|
322
|
-
run_mutations_sequential(mutations, baseline_result)
|
|
323
|
-
end
|
|
324
|
-
@progress_bar&.finish
|
|
325
|
-
result
|
|
326
|
-
end
|
|
327
|
-
|
|
328
|
-
def run_mutations_sequential(mutations, baseline_result = nil)
|
|
329
|
-
integration = build_integration
|
|
330
|
-
spec_resolver = baseline_result&.failed? ? build_neutralization_resolver : nil
|
|
331
|
-
results = []
|
|
332
|
-
survived_count = 0
|
|
333
|
-
truncated = false
|
|
334
|
-
|
|
335
|
-
mutations.each_with_index do |mutation, index|
|
|
336
|
-
result = execute_or_fetch(mutation) do
|
|
337
|
-
test_command = ->(m) { integration.call(m) }
|
|
338
|
-
isolator.call(mutation: mutation, test_command: test_command, timeout: config.timeout)
|
|
339
|
-
end
|
|
118
|
+
def equivalent_results(mutations)
|
|
119
|
+
mutations.map do |mutation|
|
|
340
120
|
mutation.strip_sources!
|
|
341
|
-
|
|
342
|
-
results << result
|
|
343
|
-
survived_count += 1 if result.survived?
|
|
344
|
-
notify_result(result, index + 1)
|
|
345
|
-
|
|
346
|
-
if config.fail_fast? && survived_count >= config.fail_fast
|
|
347
|
-
truncated = true
|
|
348
|
-
break
|
|
349
|
-
end
|
|
121
|
+
Evilution::Result::MutationResult.new(mutation: mutation, status: :equivalent, duration: 0.0)
|
|
350
122
|
end
|
|
351
|
-
|
|
352
|
-
[results, truncated]
|
|
353
123
|
end
|
|
354
124
|
|
|
355
|
-
def
|
|
356
|
-
|
|
357
|
-
pool = Evilution::Parallel::Pool.new(size: config.jobs, hooks: @hooks, item_timeout: config.timeout ? config.timeout * 2 : nil)
|
|
358
|
-
worker_isolator = build_isolator
|
|
359
|
-
spec_resolver = baseline_result&.failed? ? build_neutralization_resolver : nil
|
|
360
|
-
state = { results: [], survived_count: 0, truncated: false, completed: 0 }
|
|
361
|
-
|
|
362
|
-
all_worker_stats = []
|
|
363
|
-
|
|
364
|
-
mutations.each_slice(config.jobs) do |batch|
|
|
365
|
-
break if state[:truncated]
|
|
366
|
-
|
|
367
|
-
batch_results = run_parallel_batch(batch, pool, worker_isolator, integration)
|
|
368
|
-
all_worker_stats.concat(pool.worker_stats)
|
|
369
|
-
process_batch(batch_results, baseline_result, spec_resolver, state)
|
|
370
|
-
end
|
|
371
|
-
|
|
372
|
-
log_worker_stats(aggregate_worker_stats(all_worker_stats))
|
|
373
|
-
|
|
374
|
-
[state[:results], state[:truncated]]
|
|
125
|
+
def baseline_runner
|
|
126
|
+
@baseline_runner ||= Evilution::Runner::BaselineRunner.new(config, hooks: @hooks)
|
|
375
127
|
end
|
|
376
128
|
|
|
377
|
-
def
|
|
378
|
-
|
|
379
|
-
worker_results = run_uncached_workers(batch, uncached_indices, pool, worker_isolator, integration)
|
|
380
|
-
compact_results = merge_parallel_results(batch, uncached_indices, cached_results, worker_results)
|
|
381
|
-
batch.each(&:strip_sources!)
|
|
382
|
-
batch_results = rebuild_results(batch, compact_results)
|
|
383
|
-
batch_results.each { |r| store_cached_result(r.mutation, r) }
|
|
384
|
-
batch_results
|
|
129
|
+
def diagnostics
|
|
130
|
+
@diagnostics ||= Evilution::Runner::Diagnostics.new(config)
|
|
385
131
|
end
|
|
386
132
|
|
|
387
|
-
def
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
end
|
|
397
|
-
|
|
398
|
-
def process_batch(batch_results, baseline_result, spec_resolver, state)
|
|
399
|
-
batch_results.each do |result|
|
|
400
|
-
result = neutralize_if_baseline_failed(result, baseline_result, spec_resolver)
|
|
401
|
-
state[:results] << result
|
|
402
|
-
state[:survived_count] += 1 if result.survived?
|
|
403
|
-
state[:completed] += 1
|
|
404
|
-
notify_result(result, state[:completed])
|
|
405
|
-
end
|
|
406
|
-
|
|
407
|
-
log_memory("after batch", "#{state[:completed]} complete")
|
|
408
|
-
state[:truncated] = true if should_truncate?(state[:survived_count])
|
|
409
|
-
end
|
|
410
|
-
|
|
411
|
-
def neutralize_if_baseline_failed(result, baseline_result, spec_resolver)
|
|
412
|
-
return result unless result.survived? && baseline_result && baseline_result.failed?
|
|
413
|
-
|
|
414
|
-
if config.spec_files.any?
|
|
415
|
-
neutralize = true
|
|
416
|
-
else
|
|
417
|
-
spec_file = spec_resolver.call(result.mutation.file_path) || neutralization_fallback_dir
|
|
418
|
-
neutralize = baseline_result.failed_spec_files.include?(spec_file)
|
|
419
|
-
end
|
|
420
|
-
return result unless neutralize
|
|
421
|
-
|
|
422
|
-
Evilution::Result::MutationResult.new(
|
|
423
|
-
mutation: result.mutation,
|
|
424
|
-
status: :neutral,
|
|
425
|
-
duration: result.duration,
|
|
426
|
-
test_command: result.test_command,
|
|
427
|
-
child_rss_kb: result.child_rss_kb,
|
|
428
|
-
memory_delta_kb: result.memory_delta_kb,
|
|
429
|
-
parent_rss_kb: result.parent_rss_kb,
|
|
430
|
-
error_message: result.error_message,
|
|
431
|
-
error_class: result.error_class,
|
|
432
|
-
error_backtrace: result.error_backtrace
|
|
133
|
+
def mutation_executor
|
|
134
|
+
@mutation_executor ||= Evilution::Runner::MutationExecutor.new(
|
|
135
|
+
config,
|
|
136
|
+
isolator: isolator,
|
|
137
|
+
baseline_runner: baseline_runner,
|
|
138
|
+
cache: cache,
|
|
139
|
+
hooks: @hooks,
|
|
140
|
+
diagnostics: diagnostics,
|
|
141
|
+
on_result: on_result
|
|
433
142
|
)
|
|
434
143
|
end
|
|
435
144
|
|
|
436
|
-
def
|
|
437
|
-
|
|
438
|
-
status: result.status,
|
|
439
|
-
duration: result.duration,
|
|
440
|
-
killing_test: result.killing_test,
|
|
441
|
-
test_command: result.test_command,
|
|
442
|
-
child_rss_kb: result.child_rss_kb,
|
|
443
|
-
memory_delta_kb: result.memory_delta_kb,
|
|
444
|
-
parent_rss_kb: result.parent_rss_kb,
|
|
445
|
-
error_message: result.error_message,
|
|
446
|
-
error_class: result.error_class,
|
|
447
|
-
error_backtrace: result.error_backtrace
|
|
448
|
-
}
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
def rebuild_results(batch, compact_results)
|
|
452
|
-
batch.zip(compact_results).map do |mutation, data|
|
|
453
|
-
Evilution::Result::MutationResult.new(
|
|
454
|
-
mutation: mutation,
|
|
455
|
-
status: data[:status],
|
|
456
|
-
duration: data[:duration],
|
|
457
|
-
killing_test: data[:killing_test],
|
|
458
|
-
test_command: data[:test_command],
|
|
459
|
-
child_rss_kb: data[:child_rss_kb],
|
|
460
|
-
memory_delta_kb: data[:memory_delta_kb],
|
|
461
|
-
parent_rss_kb: data[:parent_rss_kb],
|
|
462
|
-
error_message: data[:error_message],
|
|
463
|
-
error_class: data[:error_class],
|
|
464
|
-
error_backtrace: data[:error_backtrace]
|
|
465
|
-
)
|
|
466
|
-
end
|
|
145
|
+
def run_baseline(subjects)
|
|
146
|
+
baseline_runner.call(subjects)
|
|
467
147
|
end
|
|
468
148
|
|
|
469
|
-
def
|
|
470
|
-
|
|
149
|
+
def run_mutations(mutations, baseline_result = nil)
|
|
150
|
+
mutation_executor.call(mutations, baseline_result)
|
|
471
151
|
end
|
|
472
152
|
|
|
473
153
|
def install_signal_handlers
|
|
@@ -490,336 +170,19 @@ class Evilution::Runner
|
|
|
490
170
|
end
|
|
491
171
|
end
|
|
492
172
|
|
|
493
|
-
def
|
|
494
|
-
|
|
495
|
-
when :fork then Evilution::Isolation::Fork.new(hooks: @hooks)
|
|
496
|
-
when :in_process then Evilution::Isolation::InProcess.new
|
|
497
|
-
end
|
|
498
|
-
end
|
|
499
|
-
|
|
500
|
-
def resolve_isolation
|
|
501
|
-
case config.isolation
|
|
502
|
-
when :fork
|
|
503
|
-
:fork
|
|
504
|
-
when :in_process
|
|
505
|
-
warn_in_process_under_rails if rails_root_detected?
|
|
506
|
-
:in_process
|
|
507
|
-
else # :auto
|
|
508
|
-
rails_root_detected? ? :fork : :in_process
|
|
509
|
-
end
|
|
510
|
-
end
|
|
511
|
-
|
|
512
|
-
def rails_root_detected?
|
|
513
|
-
return @rails_root_detected if defined?(@rails_root_detected)
|
|
514
|
-
|
|
515
|
-
@rails_root_detected = !detected_rails_root.nil?
|
|
516
|
-
end
|
|
517
|
-
|
|
518
|
-
def detected_rails_root
|
|
519
|
-
return @detected_rails_root if defined?(@detected_rails_root)
|
|
520
|
-
|
|
521
|
-
@detected_rails_root = Evilution::RailsDetector.rails_root_for_any(resolve_target_files)
|
|
522
|
-
end
|
|
523
|
-
|
|
524
|
-
def perform_preload
|
|
525
|
-
return if config.preload == false
|
|
526
|
-
return unless resolve_isolation == :fork
|
|
527
|
-
|
|
528
|
-
path = resolve_preload_path
|
|
529
|
-
return unless path
|
|
530
|
-
|
|
531
|
-
prepare_load_path_for_preload
|
|
532
|
-
require File.expand_path(path)
|
|
533
|
-
rescue ScriptError, StandardError => e
|
|
534
|
-
raise Evilution::ConfigError.new(
|
|
535
|
-
"failed to preload #{path.inspect}: #{e.class}: #{e.message}",
|
|
536
|
-
file: path
|
|
537
|
-
)
|
|
538
|
-
end
|
|
539
|
-
|
|
540
|
-
# Preload files (e.g. spec/rails_helper.rb) typically `require 'spec_helper'`
|
|
541
|
-
# which needs spec/ on $LOAD_PATH, and use `RSpec.configure` which needs
|
|
542
|
-
# rspec/core loaded. The RSpec CLI normally sets this up, but evilution
|
|
543
|
-
# calls Runner.run directly.
|
|
544
|
-
def prepare_load_path_for_preload
|
|
545
|
-
spec_dir = File.expand_path(resolve_spec_dir)
|
|
546
|
-
$LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
|
|
547
|
-
require "rspec/core" if config.integration == :rspec
|
|
548
|
-
end
|
|
549
|
-
|
|
550
|
-
def resolve_spec_dir
|
|
551
|
-
root = detected_rails_root
|
|
552
|
-
return File.join(root, "spec") if root
|
|
553
|
-
|
|
554
|
-
"spec"
|
|
555
|
-
end
|
|
556
|
-
|
|
557
|
-
def resolve_preload_path
|
|
558
|
-
if config.preload.is_a?(String)
|
|
559
|
-
unless File.file?(config.preload)
|
|
560
|
-
raise Evilution::ConfigError.new(
|
|
561
|
-
"preload file not found: #{config.preload.inspect}",
|
|
562
|
-
file: config.preload
|
|
563
|
-
)
|
|
564
|
-
end
|
|
565
|
-
return config.preload
|
|
566
|
-
end
|
|
567
|
-
|
|
568
|
-
root = detected_rails_root
|
|
569
|
-
return nil unless root
|
|
570
|
-
|
|
571
|
-
PRELOAD_CANDIDATES.each do |rel|
|
|
572
|
-
abs = File.join(root, rel)
|
|
573
|
-
return abs if File.file?(abs)
|
|
574
|
-
end
|
|
575
|
-
nil
|
|
576
|
-
end
|
|
577
|
-
|
|
578
|
-
# When the user explicitly requests InProcess on a Rails project, warn once
|
|
579
|
-
# per run. Rails wraps ActiveRecord transactions in
|
|
580
|
-
# Thread.handle_interrupt(Exception => :never), which defers Timeout's
|
|
581
|
-
# Thread#raise indefinitely — making InProcess unable to kill runaway mutants.
|
|
582
|
-
def warn_in_process_under_rails
|
|
583
|
-
return if config.quiet
|
|
584
|
-
return if @warned_in_process_under_rails
|
|
585
|
-
|
|
586
|
-
@warned_in_process_under_rails = true
|
|
587
|
-
$stderr.write(
|
|
588
|
-
"[evilution] warning: --isolation in_process is unsafe on Rails projects. " \
|
|
589
|
-
"ActiveRecord wraps transactions in Thread.handle_interrupt(Exception => :never), " \
|
|
590
|
-
"which swallows Timeout.timeout and can cause evilution to hang indefinitely on " \
|
|
591
|
-
"mutants that introduce infinite loops. Use --isolation fork for reliable interruption.\n"
|
|
592
|
-
)
|
|
593
|
-
end
|
|
594
|
-
|
|
595
|
-
def resolve_integration_class
|
|
596
|
-
INTEGRATIONS.fetch(config.integration) do
|
|
597
|
-
raise Evilution::Error, "unknown integration: #{config.integration}"
|
|
598
|
-
end
|
|
599
|
-
end
|
|
600
|
-
|
|
601
|
-
def build_integration
|
|
602
|
-
klass = resolve_integration_class
|
|
603
|
-
test_files = config.spec_files.empty? ? nil : config.spec_files
|
|
604
|
-
kwargs = { test_files: test_files, hooks: @hooks }
|
|
605
|
-
kwargs[:related_specs_heuristic] = config.related_specs_heuristic? if klass == Evilution::Integration::RSpec
|
|
606
|
-
klass.new(**kwargs)
|
|
607
|
-
end
|
|
608
|
-
|
|
609
|
-
def build_neutralization_resolver
|
|
610
|
-
options = resolve_integration_class.baseline_options
|
|
611
|
-
options[:spec_resolver] || Evilution::SpecResolver.new
|
|
612
|
-
end
|
|
613
|
-
|
|
614
|
-
def neutralization_fallback_dir
|
|
615
|
-
options = resolve_integration_class.baseline_options
|
|
616
|
-
options[:fallback_dir] || "spec"
|
|
173
|
+
def report_publisher
|
|
174
|
+
@report_publisher ||= Evilution::Runner::ReportPublisher.new(config)
|
|
617
175
|
end
|
|
618
176
|
|
|
619
177
|
def output_report(summary)
|
|
620
|
-
|
|
621
|
-
return unless reporter
|
|
622
|
-
|
|
623
|
-
output = reporter.call(summary)
|
|
624
|
-
return if config.quiet
|
|
625
|
-
|
|
626
|
-
if config.html?
|
|
627
|
-
path = "evilution-report.html"
|
|
628
|
-
File.write(path, output)
|
|
629
|
-
warn "HTML report written to #{path}"
|
|
630
|
-
else
|
|
631
|
-
$stdout.puts(output)
|
|
632
|
-
end
|
|
633
|
-
end
|
|
634
|
-
|
|
635
|
-
def log_baseline_start
|
|
636
|
-
return if config.quiet || !config.text? || !$stderr.tty?
|
|
637
|
-
|
|
638
|
-
$stderr.write("Running baseline test suite...\n")
|
|
639
|
-
end
|
|
640
|
-
|
|
641
|
-
def log_baseline_complete(result)
|
|
642
|
-
return if config.quiet || !config.text? || !$stderr.tty?
|
|
643
|
-
|
|
644
|
-
count = result.failed_spec_files.size
|
|
645
|
-
$stderr.write("Baseline complete: #{count} failing spec file#{"s" unless count == 1}\n")
|
|
646
|
-
end
|
|
647
|
-
|
|
648
|
-
def log_progress(current, status)
|
|
649
|
-
return if config.quiet || !config.text? || !$stderr.tty?
|
|
650
|
-
|
|
651
|
-
$stderr.write("mutation #{current} #{status}\n")
|
|
652
|
-
end
|
|
653
|
-
|
|
654
|
-
def log_memory(phase, context = nil)
|
|
655
|
-
return unless config.verbose && !config.quiet
|
|
656
|
-
|
|
657
|
-
rss = Evilution::Memory.rss_mb
|
|
658
|
-
return unless rss
|
|
659
|
-
|
|
660
|
-
gc = gc_stats_string
|
|
661
|
-
msg = format("[memory] %<phase>s: %<rss>.1f MB", phase: phase, rss: rss)
|
|
662
|
-
context = [context, gc].compact.join(", ")
|
|
663
|
-
msg += " (#{context})" unless context.empty?
|
|
664
|
-
$stderr.write("#{msg}\n")
|
|
665
|
-
end
|
|
666
|
-
|
|
667
|
-
def log_mutation_diagnostics(result)
|
|
668
|
-
return unless config.verbose && !config.quiet
|
|
669
|
-
|
|
670
|
-
parts = []
|
|
671
|
-
parts << format("child_rss: %<mb>.1f MB", mb: result.child_rss_kb / 1024.0) if result.child_rss_kb
|
|
672
|
-
|
|
673
|
-
if result.memory_delta_kb
|
|
674
|
-
sign = result.memory_delta_kb.negative? ? "" : "+"
|
|
675
|
-
parts << format("delta: %<sign>s%<mb>.1f MB", sign: sign, mb: result.memory_delta_kb / 1024.0)
|
|
676
|
-
end
|
|
677
|
-
|
|
678
|
-
parts << gc_stats_string
|
|
679
|
-
|
|
680
|
-
$stderr.write("[verbose] #{result.mutation}: #{parts.join(", ")}\n") unless parts.empty?
|
|
681
|
-
|
|
682
|
-
log_mutation_error(result) if result.error?
|
|
683
|
-
end
|
|
684
|
-
|
|
685
|
-
def log_mutation_error(result)
|
|
686
|
-
header = "[verbose] #{result.mutation}: error"
|
|
687
|
-
header += " #{result.error_class}" if result.error_class
|
|
688
|
-
header += ": #{result.error_message}" if result.error_message
|
|
689
|
-
$stderr.write("#{header}\n")
|
|
690
|
-
|
|
691
|
-
Array(result.error_backtrace).first(5).each do |line|
|
|
692
|
-
$stderr.write("[verbose] #{line}\n")
|
|
693
|
-
end
|
|
694
|
-
end
|
|
695
|
-
|
|
696
|
-
def gc_stats_string
|
|
697
|
-
stats = GC.stat
|
|
698
|
-
format(
|
|
699
|
-
"heap_live_slots: %<live>d, allocated: %<alloc>d, freed: %<freed>d",
|
|
700
|
-
live: stats[:heap_live_slots],
|
|
701
|
-
alloc: stats[:total_allocated_objects],
|
|
702
|
-
freed: stats[:total_freed_objects]
|
|
703
|
-
)
|
|
178
|
+
report_publisher.publish(summary)
|
|
704
179
|
end
|
|
705
180
|
|
|
706
181
|
def save_session(summary)
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
Evilution::Session::Store.new.save(summary)
|
|
710
|
-
rescue StandardError => e
|
|
711
|
-
warn "[evilution] failed to save session: #{e.message}" unless config.quiet
|
|
712
|
-
end
|
|
713
|
-
|
|
714
|
-
def log_worker_stats(stats)
|
|
715
|
-
return unless config.verbose && !config.quiet && stats.any?
|
|
716
|
-
|
|
717
|
-
stats.each do |stat|
|
|
718
|
-
pct = format("%.1f", stat.utilization * 100)
|
|
719
|
-
$stderr.write("[verbose] worker #{stat.pid}: #{stat.items_completed} items, utilization #{pct}%\n")
|
|
720
|
-
end
|
|
721
|
-
end
|
|
722
|
-
|
|
723
|
-
def aggregate_worker_stats(stats)
|
|
724
|
-
return stats if stats.empty?
|
|
725
|
-
|
|
726
|
-
stats.group_by(&:pid).map do |pid, entries|
|
|
727
|
-
Evilution::Parallel::WorkQueue::WorkerStat.new(
|
|
728
|
-
pid,
|
|
729
|
-
entries.sum(&:items_completed),
|
|
730
|
-
entries.sum(&:busy_time),
|
|
731
|
-
entries.sum(&:wall_time)
|
|
732
|
-
)
|
|
733
|
-
end
|
|
734
|
-
end
|
|
735
|
-
|
|
736
|
-
def notify_result(result, index)
|
|
737
|
-
on_result&.call(result)
|
|
738
|
-
@progress_bar&.tick(status: result.status)
|
|
739
|
-
log_progress(index, result.status)
|
|
740
|
-
log_mutation_diagnostics(result)
|
|
741
|
-
end
|
|
742
|
-
|
|
743
|
-
def build_progress_bar(total)
|
|
744
|
-
return nil if !config.progress? || config.quiet || config.verbose || !config.text? || !$stderr.tty?
|
|
745
|
-
|
|
746
|
-
Evilution::Reporter::ProgressBar.new(total: total, output: $stderr)
|
|
182
|
+
report_publisher.save_session(summary)
|
|
747
183
|
end
|
|
748
184
|
|
|
749
|
-
def
|
|
750
|
-
|
|
751
|
-
when :json
|
|
752
|
-
Evilution::Reporter::JSON.new(integration: config.integration)
|
|
753
|
-
when :text
|
|
754
|
-
Evilution::Reporter::CLI.new
|
|
755
|
-
when :html
|
|
756
|
-
Evilution::Reporter::HTML.new(baseline: load_baseline_session, integration: config.integration)
|
|
757
|
-
end
|
|
758
|
-
end
|
|
759
|
-
|
|
760
|
-
def load_baseline_session
|
|
761
|
-
path = config.baseline_session
|
|
762
|
-
return nil unless path
|
|
763
|
-
|
|
764
|
-
store = Evilution::Session::Store.new
|
|
765
|
-
store.load(path)
|
|
766
|
-
end
|
|
767
|
-
|
|
768
|
-
def partition_cached(batch)
|
|
769
|
-
uncached_indices = []
|
|
770
|
-
cached_results = {}
|
|
771
|
-
|
|
772
|
-
batch.each_with_index do |mutation, i|
|
|
773
|
-
cached = fetch_cached_result(mutation)
|
|
774
|
-
if cached
|
|
775
|
-
cached_results[i] = compact_result(cached)
|
|
776
|
-
else
|
|
777
|
-
uncached_indices << i
|
|
778
|
-
end
|
|
779
|
-
end
|
|
780
|
-
|
|
781
|
-
[uncached_indices, cached_results]
|
|
782
|
-
end
|
|
783
|
-
|
|
784
|
-
def merge_parallel_results(batch, uncached_indices, cached_results, worker_results)
|
|
785
|
-
result_map = cached_results.dup
|
|
786
|
-
uncached_indices.each_with_index { |batch_idx, worker_idx| result_map[batch_idx] = worker_results[worker_idx] }
|
|
787
|
-
batch.each_index.map { |i| result_map[i] }
|
|
788
|
-
end
|
|
789
|
-
|
|
790
|
-
def execute_or_fetch(mutation)
|
|
791
|
-
cached = fetch_cached_result(mutation)
|
|
792
|
-
return cached if cached
|
|
793
|
-
|
|
794
|
-
result = yield
|
|
795
|
-
store_cached_result(mutation, result)
|
|
796
|
-
result
|
|
797
|
-
end
|
|
798
|
-
|
|
799
|
-
def fetch_cached_result(mutation)
|
|
800
|
-
return nil unless cache
|
|
801
|
-
|
|
802
|
-
data = cache.fetch(mutation)
|
|
803
|
-
return nil unless data
|
|
804
|
-
return nil unless %i[killed timeout].include?(data[:status])
|
|
805
|
-
|
|
806
|
-
Evilution::Result::MutationResult.new(
|
|
807
|
-
mutation: mutation,
|
|
808
|
-
status: data[:status],
|
|
809
|
-
duration: data[:duration],
|
|
810
|
-
killing_test: data[:killing_test],
|
|
811
|
-
test_command: data[:test_command]
|
|
812
|
-
)
|
|
813
|
-
end
|
|
814
|
-
|
|
815
|
-
def store_cached_result(mutation, result)
|
|
816
|
-
return unless cache
|
|
817
|
-
return unless result.killed? || result.timeout?
|
|
818
|
-
|
|
819
|
-
cache.store(mutation,
|
|
820
|
-
status: result.status,
|
|
821
|
-
duration: result.duration,
|
|
822
|
-
killing_test: result.killing_test,
|
|
823
|
-
test_command: result.test_command)
|
|
185
|
+
def log_memory(phase, context = nil)
|
|
186
|
+
diagnostics.log_memory(phase, context)
|
|
824
187
|
end
|
|
825
188
|
end
|