evilution 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +58 -0
- data/.rubocop_todo.yml +9 -0
- data/CHANGELOG.md +86 -0
- data/README.md +202 -14
- data/docs/architecture.md +54 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/ast/parser.rb +1 -0
- data/lib/evilution/ast/pattern/filter.rb +1 -0
- data/lib/evilution/ast/pattern/parser.rb +1 -0
- data/lib/evilution/ast/pattern.rb +2 -0
- data/lib/evilution/ast/sorbet_sig_detector.rb +1 -0
- data/lib/evilution/baseline.rb +37 -15
- data/lib/evilution/child_output.rb +1 -1
- data/lib/evilution/cli/command.rb +1 -0
- data/lib/evilution/cli/commands/tests_list.rb +4 -3
- data/lib/evilution/cli/commands.rb +2 -0
- data/lib/evilution/cli/dispatcher.rb +2 -0
- data/lib/evilution/cli/exit_guard.rb +61 -0
- data/lib/evilution/cli/parsed_args.rb +2 -0
- data/lib/evilution/cli/parser/command_extractor.rb +2 -0
- data/lib/evilution/cli/parser/file_args.rb +2 -0
- data/lib/evilution/cli/parser/options_builder.rb +4 -0
- data/lib/evilution/cli/parser/stdin_reader.rb +1 -0
- data/lib/evilution/cli/parser.rb +1 -0
- data/lib/evilution/cli/printers/tests_list.rb +5 -4
- data/lib/evilution/cli/printers.rb +2 -0
- data/lib/evilution/cli/result.rb +2 -0
- data/lib/evilution/cli.rb +6 -0
- data/lib/evilution/config/builders.rb +2 -0
- data/lib/evilution/config/env_loader.rb +2 -0
- data/lib/evilution/config/file_loader.rb +1 -0
- data/lib/evilution/config/sources.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_cache.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_fallback.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_strategy.rb +1 -0
- data/lib/evilution/config/validators/fail_fast.rb +1 -0
- data/lib/evilution/config/validators/hooks.rb +1 -0
- data/lib/evilution/config/validators/ignore_patterns.rb +1 -0
- data/lib/evilution/config/validators/integration.rb +1 -0
- data/lib/evilution/config/validators/isolation.rb +1 -0
- data/lib/evilution/config/validators/jobs.rb +1 -0
- data/lib/evilution/config/validators/preload.rb +1 -0
- data/lib/evilution/config/validators/profile.rb +1 -0
- data/lib/evilution/config/validators/spec_mappings.rb +1 -0
- data/lib/evilution/config/validators/spec_pattern.rb +1 -0
- data/lib/evilution/config/validators/warmup.rb +17 -0
- data/lib/evilution/config/validators.rb +2 -0
- data/lib/evilution/config.rb +7 -5
- data/lib/evilution/coverage.rb +1 -1
- data/lib/evilution/coverage_example_filter.rb +1 -1
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/example_filter.rb +1 -1
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +10 -2
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/loading/test_load_path.rb +21 -13
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +5 -1
- data/lib/evilution/integration/rspec/state_guard/example_groups_constants.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/internals.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/reporter_arrays.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_example_groups.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_filtered_examples.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard.rb +6 -0
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/info_tool/actions/environment.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/feedback.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/statuses.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/tests.rb +1 -0
- data/lib/evilution/mutation.rb +1 -1
- data/lib/evilution/mutator/operator/argument_list_removal.rb +52 -0
- data/lib/evilution/mutator/operator/argument_propagation.rb +54 -0
- data/lib/evilution/mutator/operator/array_coercion_to_literal.rb +34 -0
- data/lib/evilution/mutator/operator/attribute_write_to_read.rb +34 -0
- data/lib/evilution/mutator/operator/bang_method.rb +11 -3
- data/lib/evilution/mutator/operator/binary_operand_promotion.rb +63 -0
- data/lib/evilution/mutator/operator/block_body_promotion.rb +45 -0
- data/lib/evilution/mutator/operator/block_body_to_nil.rb +52 -0
- data/lib/evilution/mutator/operator/block_body_to_raise.rb +34 -0
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/call_to_nil.rb +34 -0
- data/lib/evilution/mutator/operator/coercion_emptying.rb +60 -0
- data/lib/evilution/mutator/operator/collection_replacement.rb +19 -9
- data/lib/evilution/mutator/operator/comparison_replacement.rb +37 -12
- data/lib/evilution/mutator/operator/const_get_to_constant_path.rb +50 -0
- data/lib/evilution/mutator/operator/dig_to_fetch_chain.rb +45 -0
- data/lib/evilution/mutator/operator/double_negation_removal.rb +22 -0
- data/lib/evilution/mutator/operator/dynamic_dispatch_resolution.rb +79 -0
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/inequality_to_negated_identity.rb +43 -0
- data/lib/evilution/mutator/operator/keyword_argument_removal.rb +47 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/proc_to_lambda.rb +43 -0
- data/lib/evilution/mutator/operator/receiver_constructor_swap.rb +51 -0
- data/lib/evilution/mutator/operator/reduce_to_sum.rb +58 -0
- data/lib/evilution/mutator/operator/regexp_anchor_to_predicate.rb +136 -0
- data/lib/evilution/mutator/operator/safe_navigation_removal.rb +67 -0
- data/lib/evilution/mutator/operator/send_mutation.rb +42 -6
- data/lib/evilution/mutator/operator/symbol_to_proc_replacement.rb +59 -0
- data/lib/evilution/mutator/operator/to_i_to_integer.rb +38 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +25 -0
- data/lib/evilution/mutator/registry.rb +32 -1
- data/lib/evilution/parallel/pool.rb +1 -0
- data/lib/evilution/parallel_db_warning.rb +1 -1
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/rails_warmup.rb +61 -0
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/html/report.rb +1 -0
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/coverage_gap_grouper.rb +1 -0
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +38 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/baseline_runner.rb +17 -7
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/isolation_resolver.rb +23 -2
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +23 -10
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/source_ast_cache.rb +1 -1
- data/lib/evilution/spec_ast_cache.rb +1 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +35 -0
- metadata +50 -2
|
@@ -8,8 +8,9 @@ require_relative "../reporter"
|
|
|
8
8
|
require_relative "../session/schema"
|
|
9
9
|
|
|
10
10
|
class Evilution::Reporter::JSON
|
|
11
|
-
def initialize(suggest_tests: false, integration: :rspec)
|
|
11
|
+
def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new)
|
|
12
12
|
@suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
|
|
13
|
+
@subjects = subjects
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
def call(summary)
|
|
@@ -25,6 +26,7 @@ class Evilution::Reporter::JSON
|
|
|
25
26
|
timestamp: Time.now.iso8601,
|
|
26
27
|
summary: build_summary(summary),
|
|
27
28
|
coverage_gaps: build_coverage_gaps(summary),
|
|
29
|
+
subjects: @subjects.call(summary),
|
|
28
30
|
**result_categories(summary)
|
|
29
31
|
}
|
|
30
32
|
append_disabled_to_report(report, summary)
|
|
@@ -94,12 +96,20 @@ class Evilution::Reporter::JSON
|
|
|
94
96
|
end
|
|
95
97
|
|
|
96
98
|
def append_optional_summary_fields(data, summary)
|
|
99
|
+
append_diagnostic_summary_fields(data, summary)
|
|
97
100
|
data[:truncated] = true if summary.truncated?
|
|
98
101
|
data[:skipped] = summary.skipped if summary.skipped.positive?
|
|
99
102
|
peak = summary.peak_memory_mb
|
|
100
103
|
data[:peak_memory_mb] = peak.round(1) if peak
|
|
101
104
|
end
|
|
102
105
|
|
|
106
|
+
# What the run had to say about itself rather than about the mutations: a
|
|
107
|
+
# target that was never tested, work that had to be redone serially.
|
|
108
|
+
def append_diagnostic_summary_fields(data, summary)
|
|
109
|
+
data[:unresolved_target_files] = summary.unresolved_target_files if summary.unresolved_targets?
|
|
110
|
+
data[:infra_retried] = summary.infra_retried if summary.infra_retried.positive?
|
|
111
|
+
end
|
|
112
|
+
|
|
103
113
|
def build_mutation_detail(result)
|
|
104
114
|
mutation = result.mutation
|
|
105
115
|
detail = base_mutation_fields(mutation, result)
|
|
@@ -107,9 +117,19 @@ class Evilution::Reporter::JSON
|
|
|
107
117
|
detail[:test_command] = result.test_command if result.test_command
|
|
108
118
|
append_memory_fields(detail, result)
|
|
109
119
|
append_error_fields(detail, result)
|
|
120
|
+
append_neutral_reason(detail, result)
|
|
110
121
|
detail
|
|
111
122
|
end
|
|
112
123
|
|
|
124
|
+
# Why a neutral was recorded — a red baseline and an infrastructure crash want
|
|
125
|
+
# opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
|
|
126
|
+
def append_neutral_reason(detail, result)
|
|
127
|
+
reason = result.neutral_reason
|
|
128
|
+
return unless reason
|
|
129
|
+
|
|
130
|
+
detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
|
|
131
|
+
end
|
|
132
|
+
|
|
113
133
|
def base_mutation_fields(mutation, result)
|
|
114
134
|
{
|
|
115
135
|
operator: mutation.operator_name,
|
|
@@ -161,3 +181,5 @@ class Evilution::Reporter::JSON
|
|
|
161
181
|
}
|
|
162
182
|
end
|
|
163
183
|
end
|
|
184
|
+
|
|
185
|
+
require_relative "json/subjects"
|
|
@@ -7,10 +7,10 @@ require_relative "memory_stats"
|
|
|
7
7
|
class Evilution::Result::MutationResult
|
|
8
8
|
STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
|
|
9
9
|
|
|
10
|
-
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error
|
|
10
|
+
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
|
|
11
11
|
|
|
12
12
|
def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
|
|
13
|
-
test_command: nil, memory: nil, error: nil)
|
|
13
|
+
test_command: nil, memory: nil, error: nil, neutral_reason: nil)
|
|
14
14
|
raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
|
|
15
15
|
|
|
16
16
|
@mutation = mutation
|
|
@@ -20,6 +20,7 @@ class Evilution::Result::MutationResult
|
|
|
20
20
|
@test_command = test_command
|
|
21
21
|
@memory = memory
|
|
22
22
|
@error = error
|
|
23
|
+
@neutral_reason = neutral_reason
|
|
23
24
|
freeze
|
|
24
25
|
end
|
|
25
26
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
# Why a mutation was recorded neutral.
|
|
6
|
+
#
|
|
7
|
+
# Neutral covers two unrelated situations, and they call for opposite responses:
|
|
8
|
+
# a spec file that was already red before any mutation ran, and a test process
|
|
9
|
+
# that died on infrastructure rather than on the mutation. Without the reason
|
|
10
|
+
# they are indistinguishable in a report, and a run can print full marks while
|
|
11
|
+
# the neutral bucket quietly holds what would otherwise be survivors
|
|
12
|
+
# (EV-5pob / GH #1606).
|
|
13
|
+
Evilution::Result::NeutralReason = Data.define(:kind, :detail) do
|
|
14
|
+
def self.baseline_failure(spec_file)
|
|
15
|
+
new(kind: :baseline_failure, detail: spec_file)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.infra_error(error_class)
|
|
19
|
+
new(kind: :infra_error, detail: error_class)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def to_s
|
|
23
|
+
detail ? "#{label} (#{detail})" : label
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def label
|
|
29
|
+
case kind
|
|
30
|
+
when :baseline_failure then "baseline already failing"
|
|
31
|
+
when :infra_error then "infrastructure error"
|
|
32
|
+
else kind.to_s
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
# What one subject — a single method — scored, alongside the counts behind it.
|
|
6
|
+
#
|
|
7
|
+
# The run's own score is computed per file, so a well-tested file that gains new
|
|
8
|
+
# untested methods still reports 100%: whatever the resolved spec reaches is all
|
|
9
|
+
# the number ever describes (EV-nlx1 / GH #1605). Per subject the picture
|
|
10
|
+
# separates: a method whose mutations were never reached scores nothing and says
|
|
11
|
+
# so, instead of disappearing into the file's total.
|
|
12
|
+
Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived) do
|
|
13
|
+
# Killed over what actually got a verdict, the same denominator the run uses.
|
|
14
|
+
def score
|
|
15
|
+
return 0.0 if verified.zero?
|
|
16
|
+
|
|
17
|
+
killed.to_f / verified
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Whether any mutation of this subject got a verdict at all.
|
|
21
|
+
def reached?
|
|
22
|
+
verified.positive?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def fully_verified?
|
|
26
|
+
reached? && killed == verified
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
require_relative "subject_score"
|
|
5
|
+
|
|
6
|
+
# Groups mutation results by the subject they belong to and scores each one.
|
|
7
|
+
#
|
|
8
|
+
# Same denominator as the run's own score: statuses that carry no verdict
|
|
9
|
+
# (unresolved, neutral, equivalent, errored, unparseable) are left out of it, so
|
|
10
|
+
# a subject those account for entirely reports as unreached rather than as a
|
|
11
|
+
# perfect score over nothing.
|
|
12
|
+
class Evilution::Result::SubjectScorer
|
|
13
|
+
VERDICT_STATUSES = %i[killed survived timeout].freeze
|
|
14
|
+
private_constant :VERDICT_STATUSES
|
|
15
|
+
|
|
16
|
+
def call(results)
|
|
17
|
+
grouped = results.group_by { |result| [result.mutation.file_path, result.mutation.subject.name] }
|
|
18
|
+
|
|
19
|
+
grouped
|
|
20
|
+
.map { |(file_path, name), subject_results| score_for(name, file_path, subject_results) }
|
|
21
|
+
.sort_by { |score| [score.file_path, score.name] }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def score_for(name, file_path, results)
|
|
27
|
+
verdicts = results.count { |result| VERDICT_STATUSES.include?(result.status) }
|
|
28
|
+
|
|
29
|
+
Evilution::Result::SubjectScore.new(
|
|
30
|
+
name: name,
|
|
31
|
+
file_path: file_path,
|
|
32
|
+
total: results.length,
|
|
33
|
+
killed: results.count(&:killed?),
|
|
34
|
+
verified: verdicts,
|
|
35
|
+
survived: results.count(&:survived?)
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -2,19 +2,57 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../result"
|
|
4
4
|
require_relative "coverage_gap_grouper"
|
|
5
|
+
require_relative "subject_scorer"
|
|
5
6
|
|
|
6
7
|
class Evilution::Result::Summary
|
|
7
|
-
attr_reader :results, :duration, :skipped, :disabled_mutations
|
|
8
|
+
attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
|
|
9
|
+
:target_file_count, :infra_retried
|
|
8
10
|
|
|
9
|
-
def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: []
|
|
11
|
+
def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
|
|
12
|
+
unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
|
|
10
13
|
@results = results
|
|
11
14
|
@duration = duration
|
|
12
15
|
@truncated = truncated
|
|
13
16
|
@skipped = skipped
|
|
14
17
|
@disabled_mutations = disabled_mutations
|
|
18
|
+
@unresolved_target_files = unresolved_target_files.freeze
|
|
19
|
+
@target_file_count = target_file_count
|
|
20
|
+
@infra_retried = infra_retried
|
|
15
21
|
freeze
|
|
16
22
|
end
|
|
17
23
|
|
|
24
|
+
# Neutral results gathered by the reason they were recorded, which is what the
|
|
25
|
+
# report needs to say which kind of neutral a reader is looking at
|
|
26
|
+
# (EV-5pob / GH #1606).
|
|
27
|
+
def neutral_results_by_reason
|
|
28
|
+
neutral_results.group_by(&:neutral_reason).to_a
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# What each subject — each method — scored on its own. The run's score is
|
|
32
|
+
# computed per file, which says nothing about a method inside it that no
|
|
33
|
+
# example reaches (EV-nlx1 / GH #1605).
|
|
34
|
+
def subject_scores
|
|
35
|
+
Evilution::Result::SubjectScorer.new.call(results)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The subjects the file-level score does not speak for: something survived, or
|
|
39
|
+
# nothing reached them at all.
|
|
40
|
+
def subjects_needing_attention
|
|
41
|
+
subject_scores.reject(&:fully_verified?)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The same subjects, gathered under the file they live in, which is how the
|
|
45
|
+
# report lists them.
|
|
46
|
+
def subjects_needing_attention_by_file
|
|
47
|
+
subjects_needing_attention.group_by(&:file_path).values
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Files evilution was pointed at that resolved to no test file, so nothing
|
|
51
|
+
# about them was ever measured (EV-p4sm / GH #1603).
|
|
52
|
+
def unresolved_targets?
|
|
53
|
+
!unresolved_target_files.empty?
|
|
54
|
+
end
|
|
55
|
+
|
|
18
56
|
def truncated?
|
|
19
57
|
@truncated
|
|
20
58
|
end
|
|
@@ -66,7 +104,11 @@ class Evilution::Result::Summary
|
|
|
66
104
|
killed.to_f / denominator
|
|
67
105
|
end
|
|
68
106
|
|
|
107
|
+
# A target file that was never tested fails the run on its own: the score
|
|
108
|
+
# only speaks for the files that did resolve to a spec.
|
|
69
109
|
def success?(min_score: 1.0)
|
|
110
|
+
return false if unresolved_targets?
|
|
111
|
+
|
|
70
112
|
score >= min_score
|
|
71
113
|
end
|
|
72
114
|
|
|
@@ -60,18 +60,15 @@ class Evilution::Runner::BaselineRunner
|
|
|
60
60
|
return nil unless config.baseline? && subjects.any?
|
|
61
61
|
|
|
62
62
|
log_start
|
|
63
|
-
|
|
64
|
-
timeout: config.timeout,
|
|
65
|
-
test_files: config.spec_files.empty? ? nil : config.spec_files,
|
|
66
|
-
**integration_class.baseline_options
|
|
67
|
-
)
|
|
68
|
-
result = baseline.call(subjects)
|
|
63
|
+
result = build_baseline.call(subjects)
|
|
69
64
|
log_complete(result)
|
|
70
65
|
result
|
|
71
66
|
end
|
|
72
67
|
|
|
68
|
+
# The same selector the baseline resolves with, so a surviving mutation is
|
|
69
|
+
# matched against exactly the spec files the baseline ran.
|
|
73
70
|
def neutralization_resolver
|
|
74
|
-
|
|
71
|
+
config.spec_selector
|
|
75
72
|
end
|
|
76
73
|
|
|
77
74
|
def neutralization_fallback_dir
|
|
@@ -82,6 +79,19 @@ class Evilution::Runner::BaselineRunner
|
|
|
82
79
|
|
|
83
80
|
attr_reader :config, :hooks
|
|
84
81
|
|
|
82
|
+
# Resolves through config.spec_selector -- the object `run` uses, so
|
|
83
|
+
# spec_mappings / spec_pattern apply -- and skips unresolved sources unless
|
|
84
|
+
# the run falls back to the full suite.
|
|
85
|
+
def build_baseline
|
|
86
|
+
Evilution::Baseline.new(
|
|
87
|
+
timeout: config.timeout,
|
|
88
|
+
test_files: config.spec_files.empty? ? nil : config.spec_files,
|
|
89
|
+
spec_selector: config.spec_selector,
|
|
90
|
+
fallback_to_full_suite: config.fallback_to_full_suite?,
|
|
91
|
+
**integration_class.baseline_options
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
85
95
|
def build_example_filter
|
|
86
96
|
return nil unless config.example_targeting?
|
|
87
97
|
|
|
@@ -34,7 +34,7 @@ class Evilution::Runner::Canary
|
|
|
34
34
|
test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
|
|
35
35
|
timeout: @config.timeout
|
|
36
36
|
)
|
|
37
|
-
raise Failed, failure_message(result
|
|
37
|
+
raise Failed, failure_message(result) unless result.status == :survived
|
|
38
38
|
|
|
39
39
|
nil
|
|
40
40
|
ensure
|
|
@@ -146,12 +146,59 @@ class Evilution::Runner::Canary
|
|
|
146
146
|
@integration_class.new(test_files: [spec_path], hooks: @hooks)
|
|
147
147
|
end
|
|
148
148
|
|
|
149
|
-
|
|
149
|
+
# When the child reported an error, that error IS the diagnosis -- naming it
|
|
150
|
+
# beats guessing. The speculative list stays only for the cases that carry no
|
|
151
|
+
# error at all (:killed, :timeout), where guesses are the only help there is.
|
|
152
|
+
# Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
|
|
153
|
+
# this message used to drop, and none of the four guesses was the cause.
|
|
154
|
+
# EV-65nf / GH #1586.
|
|
155
|
+
def failure_message(result)
|
|
156
|
+
"#{failure_preamble(result.status)} #{diagnosis(result)} " \
|
|
157
|
+
"Re-run with --no-canary to bypass this check."
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def failure_preamble(status)
|
|
150
161
|
"evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
|
|
151
162
|
"mutation was scored #{status.inspect} instead of :survived. The mutation " \
|
|
152
|
-
"pipeline is misreporting — every score this run would produce is unreliable.
|
|
153
|
-
|
|
163
|
+
"pipeline is misreporting — every score this run would produce is unreliable."
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def diagnosis(result)
|
|
167
|
+
reported = reported_error(result)
|
|
168
|
+
return speculative_causes if reported.nil?
|
|
169
|
+
|
|
170
|
+
"The child reported: #{reported}"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def reported_error(result)
|
|
174
|
+
message = result.error_message
|
|
175
|
+
return nil if message.nil? || message.empty?
|
|
176
|
+
|
|
177
|
+
described = with_class_prefix(message, result.error_class)
|
|
178
|
+
frame = first_frame(result)
|
|
179
|
+
frame.nil? ? described : "#{described} (at #{frame})"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# MutationApplier already prefixes the class onto the message it packs
|
|
183
|
+
# ("#{e.class}: #{e.message}"), while other paths pack the bare message and
|
|
184
|
+
# leave the class in its own field. Prefixing unconditionally would print
|
|
185
|
+
# "NameError: NameError: ..." for the former.
|
|
186
|
+
def with_class_prefix(message, klass)
|
|
187
|
+
return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
|
|
188
|
+
|
|
189
|
+
"#{klass}: #{message}"
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def first_frame(result)
|
|
193
|
+
backtrace = result.error_backtrace
|
|
194
|
+
return nil if backtrace.nil? || backtrace.empty?
|
|
195
|
+
|
|
196
|
+
backtrace.first
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def speculative_causes
|
|
200
|
+
"Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
|
|
154
201
|
"RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
|
|
155
|
-
"an isolation-mode defect.
|
|
202
|
+
"an isolation-mode defect."
|
|
156
203
|
end
|
|
157
204
|
end
|
|
@@ -6,6 +6,7 @@ require_relative "../isolation/in_process"
|
|
|
6
6
|
require_relative "../rails_detector"
|
|
7
7
|
require_relative "../gem_detector"
|
|
8
8
|
require_relative "../integration/loading/test_load_path"
|
|
9
|
+
require_relative "../rails_warmup"
|
|
9
10
|
|
|
10
11
|
class Evilution::Runner::IsolationResolver
|
|
11
12
|
PRELOAD_CANDIDATES = [
|
|
@@ -48,6 +49,7 @@ class Evilution::Runner::IsolationResolver
|
|
|
48
49
|
prepare_load_path_for_preload(path)
|
|
49
50
|
prepare_integration_for_preload
|
|
50
51
|
require File.expand_path(path)
|
|
52
|
+
warm_up_rails
|
|
51
53
|
rescue Evilution::ConfigError
|
|
52
54
|
raise
|
|
53
55
|
rescue ScriptError, StandardError => e
|
|
@@ -61,6 +63,21 @@ class Evilution::Runner::IsolationResolver
|
|
|
61
63
|
|
|
62
64
|
attr_reader :config, :hooks
|
|
63
65
|
|
|
66
|
+
# Runs only after a preload (which is what loads Rails). This is primarily an
|
|
67
|
+
# optimisation for :fork isolation, where children inherit the warmed state.
|
|
68
|
+
def warm_up_rails
|
|
69
|
+
return unless config.warmup == :rails
|
|
70
|
+
|
|
71
|
+
application = ::Rails.application if defined?(::Rails) && ::Rails.respond_to?(:application)
|
|
72
|
+
unless application
|
|
73
|
+
warn_message = "[evilution] warmup: rails requested but Rails is not loaded (or has no application) after preload; skipping"
|
|
74
|
+
Evilution::Diagnostic.warn(warn_message)
|
|
75
|
+
return
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
Evilution::RailsWarmup.new(application).call
|
|
79
|
+
end
|
|
80
|
+
|
|
64
81
|
# Under :fork, allow preloading — caller resolves whether a path exists (an
|
|
65
82
|
# explicit --preload / preload: value, or an auto-detected rails_helper) and
|
|
66
83
|
# bails early when none does. Under :in_process, only allow preloading when
|
|
@@ -131,7 +148,11 @@ class Evilution::Runner::IsolationResolver
|
|
|
131
148
|
# have both (a bare `require "support/foo"` must still resolve from spec/).
|
|
132
149
|
# - minitest/test-unit: a test/test_helper.rb doing a non-relative
|
|
133
150
|
# `require "support/..."` needs test/ on $LOAD_PATH. Route through
|
|
134
|
-
# TestLoadPath
|
|
151
|
+
# TestLoadPath, the policy the per-mutation test load uses, with one
|
|
152
|
+
# deliberate difference: outside_root_dirs: false. A resolved test file
|
|
153
|
+
# outside test/ and spec/ keeps its own directory, but a preload there
|
|
154
|
+
# (config/evilution_preload.rb) must not, or config/ lands ahead of the
|
|
155
|
+
# gems and config/puma.rb shadows the puma gem (GH #1597).
|
|
135
156
|
def prepare_load_path_for_preload(preload_path)
|
|
136
157
|
if config.integration == :rspec
|
|
137
158
|
prepare_rspec_preload_load_path
|
|
@@ -148,7 +169,7 @@ class Evilution::Runner::IsolationResolver
|
|
|
148
169
|
|
|
149
170
|
def prepare_test_preload_load_path(preload_path)
|
|
150
171
|
base = detected_rails_root || Evilution.project_base_dir
|
|
151
|
-
Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base)
|
|
172
|
+
Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base, outside_root_dirs: false)
|
|
152
173
|
end
|
|
153
174
|
|
|
154
175
|
def resolve_spec_dir
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../mutation_executor"
|
|
4
|
+
require_relative "neutralizer/infra_error"
|
|
5
|
+
|
|
6
|
+
# Re-runs, one at a time, the mutations a parallel pass could not judge because
|
|
7
|
+
# the test process crashed on infrastructure — a database lock, a statement
|
|
8
|
+
# timeout — rather than on the mutation itself.
|
|
9
|
+
#
|
|
10
|
+
# Those crashes are demoted to `:neutral` so they do not inflate the kill count
|
|
11
|
+
# (EV-toid / GH #814), which leaves the neutral bucket swinging with `--jobs`
|
|
12
|
+
# on identical input: the contention that causes them only exists while several
|
|
13
|
+
# workers run at once. Re-running them after the pool is done removes the
|
|
14
|
+
# contention, so the verdict matches what a serial run would have produced
|
|
15
|
+
# (EV-j0bv / GH #1607).
|
|
16
|
+
#
|
|
17
|
+
# Only mutations neutralised by an infra crash are re-run. A neutral from a
|
|
18
|
+
# failing baseline is a real statement about the spec, and re-running it would
|
|
19
|
+
# say the same thing again.
|
|
20
|
+
class Evilution::Runner::MutationExecutor::InfraRetry
|
|
21
|
+
attr_reader :retried_count
|
|
22
|
+
|
|
23
|
+
def initialize(runner:, pipeline:)
|
|
24
|
+
@runner = runner
|
|
25
|
+
@pipeline = pipeline
|
|
26
|
+
@retried_count = 0
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def call(results, baseline_result:, integration:)
|
|
30
|
+
@retried_count = 0
|
|
31
|
+
|
|
32
|
+
results.map do |result|
|
|
33
|
+
next result unless retryable?(result)
|
|
34
|
+
|
|
35
|
+
@retried_count += 1
|
|
36
|
+
rerun(result, baseline_result: baseline_result, integration: integration)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def retryable?(result)
|
|
43
|
+
Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_neutral?(result)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The parallel pass keeps the sources of a retryable mutation alive so it can
|
|
47
|
+
# be applied again; they are released here, once it has been.
|
|
48
|
+
def rerun(result, baseline_result:, integration:)
|
|
49
|
+
mutation = result.mutation
|
|
50
|
+
rerun_result = @runner.call(mutation, integration: integration)
|
|
51
|
+
mutation.strip_sources!
|
|
52
|
+
@pipeline.call(rerun_result, baseline_result: baseline_result)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../neutralizer"
|
|
4
4
|
require_relative "../../../result/mutation_result"
|
|
5
|
+
require_relative "../../../result/neutral_reason"
|
|
5
6
|
|
|
6
7
|
class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
|
|
7
8
|
def initialize(config:, spec_resolver:, fallback_dir:)
|
|
@@ -13,27 +14,39 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
|
|
|
13
14
|
def call(result, baseline_result:)
|
|
14
15
|
return result unless result.survived? && baseline_result && baseline_result.failed?
|
|
15
16
|
|
|
16
|
-
if @config.spec_files.any?
|
|
17
|
-
should_neutralize = true
|
|
18
|
-
else
|
|
19
|
-
spec_file = @spec_resolver.call(result.mutation.file_path) || @fallback_dir
|
|
20
|
-
should_neutralize = baseline_result.failed_spec_files.include?(spec_file)
|
|
21
|
-
end
|
|
22
|
-
return result unless should_neutralize
|
|
17
|
+
return neutralize(result, nil) if @config.spec_files.any?
|
|
23
18
|
|
|
24
|
-
|
|
19
|
+
failed = baseline_result.failed_spec_files
|
|
20
|
+
spec_file = covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
|
|
21
|
+
return result unless spec_file
|
|
22
|
+
|
|
23
|
+
neutralize(result, spec_file)
|
|
25
24
|
end
|
|
26
25
|
|
|
27
26
|
private
|
|
28
27
|
|
|
29
|
-
|
|
28
|
+
# The spec files the baseline ran for this source: every file the resolver
|
|
29
|
+
# returns (a spec_selector may map one source to several), or the fallback
|
|
30
|
+
# directory when nothing resolves -- but only if the run falls back to the
|
|
31
|
+
# full suite; otherwise the baseline never ran it.
|
|
32
|
+
def covering_specs(file_path)
|
|
33
|
+
specs = Array(@spec_resolver.call(file_path))
|
|
34
|
+
return specs unless specs.empty?
|
|
35
|
+
|
|
36
|
+
@config.fallback_to_full_suite? ? [@fallback_dir] : []
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The spec that was already red is the fact a reader needs: it says what to
|
|
40
|
+
# fix before the run can say anything about these mutations.
|
|
41
|
+
def neutralize(result, spec_file)
|
|
30
42
|
Evilution::Result::MutationResult.new(
|
|
31
43
|
mutation: result.mutation,
|
|
32
44
|
status: :neutral,
|
|
33
45
|
duration: result.duration,
|
|
34
46
|
test_command: result.test_command,
|
|
35
47
|
memory: result.memory,
|
|
36
|
-
error: result.error
|
|
48
|
+
error: result.error,
|
|
49
|
+
neutral_reason: Evilution::Result::NeutralReason.baseline_failure(spec_file)
|
|
37
50
|
)
|
|
38
51
|
end
|
|
39
52
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../neutralizer"
|
|
4
4
|
require_relative "../../../result/mutation_result"
|
|
5
|
+
require_relative "../../../result/neutral_reason"
|
|
5
6
|
|
|
6
7
|
# Reclassify results as :neutral when the failure was caused by test
|
|
7
8
|
# infrastructure rather than by the mutation. Two independent paths:
|
|
@@ -33,6 +34,21 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
|
|
|
33
34
|
].freeze
|
|
34
35
|
private_constant :INFRA_ERROR_CLASSES, :INFRA_BACKTRACE_PATHS, :INFRA_CRASH_CLASSES
|
|
35
36
|
|
|
37
|
+
# Whether a crash class is one of the infrastructure failures that say
|
|
38
|
+
# nothing about the mutation. The parallel strategy asks this before the
|
|
39
|
+
# neutralisation pipeline has run, while the result is still a `:killed`
|
|
40
|
+
# crash (EV-j0bv / GH #1607).
|
|
41
|
+
def self.infra_crash_class?(error_class)
|
|
42
|
+
INFRA_CRASH_CLASSES.include?(error_class)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Whether this result is a kill that was demoted because the test process
|
|
46
|
+
# crashed on infrastructure rather than on the mutation. Such a mutation got
|
|
47
|
+
# no verdict, so a parallel run can re-run it once the contention is over.
|
|
48
|
+
def self.infra_neutral?(result)
|
|
49
|
+
result.neutral? && infra_crash_class?(result.error_class)
|
|
50
|
+
end
|
|
51
|
+
|
|
36
52
|
def call(result, **_ctx)
|
|
37
53
|
return neutralize(result) if infra_crash?(result)
|
|
38
54
|
return result unless result.error?
|
|
@@ -62,7 +78,8 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
|
|
|
62
78
|
duration: result.duration,
|
|
63
79
|
test_command: result.test_command,
|
|
64
80
|
memory: result.memory,
|
|
65
|
-
error: result.error
|
|
81
|
+
error: result.error,
|
|
82
|
+
neutral_reason: Evilution::Result::NeutralReason.infra_error(result.error_class)
|
|
66
83
|
)
|
|
67
84
|
end
|
|
68
85
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../mutation_executor"
|
|
4
4
|
require_relative "../../result/mutation_result"
|
|
5
|
+
require_relative "neutralizer/infra_error"
|
|
5
6
|
|
|
6
7
|
class Evilution::Runner::MutationExecutor::ResultCache
|
|
7
8
|
CACHEABLE_STATUSES = %i[killed timeout].freeze
|
|
@@ -32,6 +33,7 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
32
33
|
def store(mutation, result)
|
|
33
34
|
return unless @backend
|
|
34
35
|
return unless result.killed? || result.timeout?
|
|
36
|
+
return if infra_crash?(result)
|
|
35
37
|
|
|
36
38
|
@backend.store(mutation,
|
|
37
39
|
status: result.status,
|
|
@@ -40,6 +42,15 @@ class Evilution::Runner::MutationExecutor::ResultCache
|
|
|
40
42
|
test_command: result.test_command)
|
|
41
43
|
end
|
|
42
44
|
|
|
45
|
+
# A kill that came from an infrastructure crash is not a verdict about the
|
|
46
|
+
# mutation, and the cache keeps no error class to tell them apart later. It
|
|
47
|
+
# is left out so the next fetch does not hand back a plain `:killed`, which
|
|
48
|
+
# would defeat both the demotion to neutral and the serial retry
|
|
49
|
+
# (EV-j0bv / GH #1607).
|
|
50
|
+
def infra_crash?(result)
|
|
51
|
+
Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
|
|
52
|
+
end
|
|
53
|
+
|
|
43
54
|
def partition(batch, packer:)
|
|
44
55
|
uncached_indices = []
|
|
45
56
|
cached_results = {}
|
|
@@ -65,10 +65,28 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
|
|
|
65
65
|
compact_results = merge(batch, partition.uncached_indices, partition.cached_results, worker_results)
|
|
66
66
|
batch_results = rebuild_results(batch, compact_results)
|
|
67
67
|
cache_results(batch_results, partition.uncached_indices)
|
|
68
|
-
batch
|
|
68
|
+
release_sources(batch, batch_results)
|
|
69
69
|
batch_results
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
+
# Sources are dropped as soon as a mutation has been judged, to keep a long
|
|
73
|
+
# run's memory flat. A mutation the workers could not judge because of
|
|
74
|
+
# infrastructure contention is re-run serially once the pool is done
|
|
75
|
+
# (InfraRetry), which needs its sources, so it keeps them until then. The
|
|
76
|
+
# neutralisation pipeline has not run yet here, so the crash is still a kill.
|
|
77
|
+
def release_sources(batch, batch_results)
|
|
78
|
+
batch.each_with_index do |mutation, index|
|
|
79
|
+
result = batch_results[index]
|
|
80
|
+
next if result && infra_crash?(result)
|
|
81
|
+
|
|
82
|
+
mutation.strip_sources!
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def infra_crash?(result)
|
|
87
|
+
Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
|
|
88
|
+
end
|
|
89
|
+
|
|
72
90
|
def rebuild_results(batch, compact_results)
|
|
73
91
|
batch.zip(compact_results).map { |m, h| @packer.rebuild(m, h) }
|
|
74
92
|
end
|