evilution 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +58 -0
  3. data/.rubocop_todo.yml +9 -0
  4. data/CHANGELOG.md +86 -0
  5. data/README.md +202 -14
  6. data/docs/architecture.md +54 -5
  7. data/docs/isolation.md +3 -4
  8. data/exe/evil +8 -1
  9. data/exe/evilution +8 -1
  10. data/lib/evilution/ast/local_reads.rb +43 -0
  11. data/lib/evilution/ast/parser.rb +1 -0
  12. data/lib/evilution/ast/pattern/filter.rb +1 -0
  13. data/lib/evilution/ast/pattern/parser.rb +1 -0
  14. data/lib/evilution/ast/pattern.rb +2 -0
  15. data/lib/evilution/ast/sorbet_sig_detector.rb +1 -0
  16. data/lib/evilution/baseline.rb +37 -15
  17. data/lib/evilution/child_output.rb +1 -1
  18. data/lib/evilution/cli/command.rb +1 -0
  19. data/lib/evilution/cli/commands/tests_list.rb +4 -3
  20. data/lib/evilution/cli/commands.rb +2 -0
  21. data/lib/evilution/cli/dispatcher.rb +2 -0
  22. data/lib/evilution/cli/exit_guard.rb +61 -0
  23. data/lib/evilution/cli/parsed_args.rb +2 -0
  24. data/lib/evilution/cli/parser/command_extractor.rb +2 -0
  25. data/lib/evilution/cli/parser/file_args.rb +2 -0
  26. data/lib/evilution/cli/parser/options_builder.rb +4 -0
  27. data/lib/evilution/cli/parser/stdin_reader.rb +1 -0
  28. data/lib/evilution/cli/parser.rb +1 -0
  29. data/lib/evilution/cli/printers/tests_list.rb +5 -4
  30. data/lib/evilution/cli/printers.rb +2 -0
  31. data/lib/evilution/cli/result.rb +2 -0
  32. data/lib/evilution/cli.rb +6 -0
  33. data/lib/evilution/config/builders.rb +2 -0
  34. data/lib/evilution/config/env_loader.rb +2 -0
  35. data/lib/evilution/config/file_loader.rb +1 -0
  36. data/lib/evilution/config/sources.rb +1 -0
  37. data/lib/evilution/config/validators/example_targeting_cache.rb +1 -0
  38. data/lib/evilution/config/validators/example_targeting_fallback.rb +1 -0
  39. data/lib/evilution/config/validators/example_targeting_strategy.rb +1 -0
  40. data/lib/evilution/config/validators/fail_fast.rb +1 -0
  41. data/lib/evilution/config/validators/hooks.rb +1 -0
  42. data/lib/evilution/config/validators/ignore_patterns.rb +1 -0
  43. data/lib/evilution/config/validators/integration.rb +1 -0
  44. data/lib/evilution/config/validators/isolation.rb +1 -0
  45. data/lib/evilution/config/validators/jobs.rb +1 -0
  46. data/lib/evilution/config/validators/preload.rb +1 -0
  47. data/lib/evilution/config/validators/profile.rb +1 -0
  48. data/lib/evilution/config/validators/spec_mappings.rb +1 -0
  49. data/lib/evilution/config/validators/spec_pattern.rb +1 -0
  50. data/lib/evilution/config/validators/warmup.rb +17 -0
  51. data/lib/evilution/config/validators.rb +2 -0
  52. data/lib/evilution/config.rb +7 -5
  53. data/lib/evilution/coverage.rb +1 -1
  54. data/lib/evilution/coverage_example_filter.rb +1 -1
  55. data/lib/evilution/diagnostic.rb +22 -0
  56. data/lib/evilution/example_filter.rb +1 -1
  57. data/lib/evilution/integration/loading/body_call_neutralizer.rb +10 -2
  58. data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
  59. data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
  60. data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
  61. data/lib/evilution/integration/loading/test_load_path.rb +21 -13
  62. data/lib/evilution/integration/minitest.rb +2 -1
  63. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
  64. data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +1 -0
  65. data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +5 -1
  66. data/lib/evilution/integration/rspec/state_guard/example_groups_constants.rb +1 -2
  67. data/lib/evilution/integration/rspec/state_guard/internals.rb +1 -2
  68. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +1 -2
  69. data/lib/evilution/integration/rspec/state_guard/reporter_arrays.rb +1 -0
  70. data/lib/evilution/integration/rspec/state_guard/world_example_groups.rb +1 -0
  71. data/lib/evilution/integration/rspec/state_guard/world_filtered_examples.rb +1 -0
  72. data/lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb +1 -0
  73. data/lib/evilution/integration/rspec/state_guard.rb +6 -0
  74. data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
  75. data/lib/evilution/integration/rspec.rb +48 -2
  76. data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
  77. data/lib/evilution/isolation/fork.rb +29 -8
  78. data/lib/evilution/mcp/info_tool/actions/environment.rb +1 -0
  79. data/lib/evilution/mcp/info_tool/actions/feedback.rb +1 -0
  80. data/lib/evilution/mcp/info_tool/actions/statuses.rb +1 -0
  81. data/lib/evilution/mcp/info_tool/actions/subjects.rb +1 -0
  82. data/lib/evilution/mcp/info_tool/actions/tests.rb +1 -0
  83. data/lib/evilution/mutation.rb +1 -1
  84. data/lib/evilution/mutator/operator/argument_list_removal.rb +52 -0
  85. data/lib/evilution/mutator/operator/argument_propagation.rb +54 -0
  86. data/lib/evilution/mutator/operator/array_coercion_to_literal.rb +34 -0
  87. data/lib/evilution/mutator/operator/attribute_write_to_read.rb +34 -0
  88. data/lib/evilution/mutator/operator/bang_method.rb +11 -3
  89. data/lib/evilution/mutator/operator/binary_operand_promotion.rb +63 -0
  90. data/lib/evilution/mutator/operator/block_body_promotion.rb +45 -0
  91. data/lib/evilution/mutator/operator/block_body_to_nil.rb +52 -0
  92. data/lib/evilution/mutator/operator/block_body_to_raise.rb +34 -0
  93. data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
  94. data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
  95. data/lib/evilution/mutator/operator/call_to_nil.rb +34 -0
  96. data/lib/evilution/mutator/operator/coercion_emptying.rb +60 -0
  97. data/lib/evilution/mutator/operator/collection_replacement.rb +19 -9
  98. data/lib/evilution/mutator/operator/comparison_replacement.rb +37 -12
  99. data/lib/evilution/mutator/operator/const_get_to_constant_path.rb +50 -0
  100. data/lib/evilution/mutator/operator/dig_to_fetch_chain.rb +45 -0
  101. data/lib/evilution/mutator/operator/double_negation_removal.rb +22 -0
  102. data/lib/evilution/mutator/operator/dynamic_dispatch_resolution.rb +79 -0
  103. data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
  104. data/lib/evilution/mutator/operator/inequality_to_negated_identity.rb +43 -0
  105. data/lib/evilution/mutator/operator/keyword_argument_removal.rb +47 -0
  106. data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
  107. data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
  108. data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
  109. data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
  110. data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
  111. data/lib/evilution/mutator/operator/proc_to_lambda.rb +43 -0
  112. data/lib/evilution/mutator/operator/receiver_constructor_swap.rb +51 -0
  113. data/lib/evilution/mutator/operator/reduce_to_sum.rb +58 -0
  114. data/lib/evilution/mutator/operator/regexp_anchor_to_predicate.rb +136 -0
  115. data/lib/evilution/mutator/operator/safe_navigation_removal.rb +67 -0
  116. data/lib/evilution/mutator/operator/send_mutation.rb +42 -6
  117. data/lib/evilution/mutator/operator/symbol_to_proc_replacement.rb +59 -0
  118. data/lib/evilution/mutator/operator/to_i_to_integer.rb +38 -0
  119. data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
  120. data/lib/evilution/mutator/primitives.rb +25 -0
  121. data/lib/evilution/mutator/registry.rb +32 -1
  122. data/lib/evilution/parallel/pool.rb +1 -0
  123. data/lib/evilution/parallel_db_warning.rb +1 -1
  124. data/lib/evilution/process_supervisor.rb +20 -8
  125. data/lib/evilution/rails_warmup.rb +61 -0
  126. data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
  127. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
  128. data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
  129. data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
  130. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
  131. data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
  132. data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
  133. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  134. data/lib/evilution/reporter/cli/trailer.rb +11 -7
  135. data/lib/evilution/reporter/cli.rb +20 -4
  136. data/lib/evilution/reporter/html/report.rb +1 -0
  137. data/lib/evilution/reporter/json/subjects.rb +29 -0
  138. data/lib/evilution/reporter/json.rb +23 -1
  139. data/lib/evilution/result/coverage_gap_grouper.rb +1 -0
  140. data/lib/evilution/result/mutation_result.rb +3 -2
  141. data/lib/evilution/result/neutral_reason.rb +35 -0
  142. data/lib/evilution/result/subject_score.rb +28 -0
  143. data/lib/evilution/result/subject_scorer.rb +38 -0
  144. data/lib/evilution/result/summary.rb +44 -2
  145. data/lib/evilution/runner/baseline_runner.rb +17 -7
  146. data/lib/evilution/runner/canary.rb +52 -5
  147. data/lib/evilution/runner/isolation_resolver.rb +23 -2
  148. data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
  149. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +23 -10
  150. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
  151. data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
  152. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
  153. data/lib/evilution/runner/mutation_executor.rb +32 -4
  154. data/lib/evilution/runner/report_publisher.rb +31 -9
  155. data/lib/evilution/runner/target_spec_audit.rb +43 -0
  156. data/lib/evilution/runner.rb +10 -1
  157. data/lib/evilution/source_ast_cache.rb +1 -1
  158. data/lib/evilution/spec_ast_cache.rb +1 -1
  159. data/lib/evilution/version.rb +1 -1
  160. data/lib/evilution.rb +35 -0
  161. metadata +50 -2
@@ -8,8 +8,9 @@ require_relative "../reporter"
8
8
  require_relative "../session/schema"
9
9
 
10
10
  class Evilution::Reporter::JSON
11
- def initialize(suggest_tests: false, integration: :rspec)
11
+ def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new)
12
12
  @suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
13
+ @subjects = subjects
13
14
  end
14
15
 
15
16
  def call(summary)
@@ -25,6 +26,7 @@ class Evilution::Reporter::JSON
25
26
  timestamp: Time.now.iso8601,
26
27
  summary: build_summary(summary),
27
28
  coverage_gaps: build_coverage_gaps(summary),
29
+ subjects: @subjects.call(summary),
28
30
  **result_categories(summary)
29
31
  }
30
32
  append_disabled_to_report(report, summary)
@@ -94,12 +96,20 @@ class Evilution::Reporter::JSON
94
96
  end
95
97
 
96
98
  def append_optional_summary_fields(data, summary)
99
+ append_diagnostic_summary_fields(data, summary)
97
100
  data[:truncated] = true if summary.truncated?
98
101
  data[:skipped] = summary.skipped if summary.skipped.positive?
99
102
  peak = summary.peak_memory_mb
100
103
  data[:peak_memory_mb] = peak.round(1) if peak
101
104
  end
102
105
 
106
+ # What the run had to say about itself rather than about the mutations: a
107
+ # target that was never tested, work that had to be redone serially.
108
+ def append_diagnostic_summary_fields(data, summary)
109
+ data[:unresolved_target_files] = summary.unresolved_target_files if summary.unresolved_targets?
110
+ data[:infra_retried] = summary.infra_retried if summary.infra_retried.positive?
111
+ end
112
+
103
113
  def build_mutation_detail(result)
104
114
  mutation = result.mutation
105
115
  detail = base_mutation_fields(mutation, result)
@@ -107,9 +117,19 @@ class Evilution::Reporter::JSON
107
117
  detail[:test_command] = result.test_command if result.test_command
108
118
  append_memory_fields(detail, result)
109
119
  append_error_fields(detail, result)
120
+ append_neutral_reason(detail, result)
110
121
  detail
111
122
  end
112
123
 
124
+ # Why a neutral was recorded — a red baseline and an infrastructure crash want
125
+ # opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
126
+ def append_neutral_reason(detail, result)
127
+ reason = result.neutral_reason
128
+ return unless reason
129
+
130
+ detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
131
+ end
132
+
113
133
  def base_mutation_fields(mutation, result)
114
134
  {
115
135
  operator: mutation.operator_name,
@@ -161,3 +181,5 @@ class Evilution::Reporter::JSON
161
181
  }
162
182
  end
163
183
  end
184
+
185
+ require_relative "json/subjects"
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "../result"
3
4
  require_relative "coverage_gap"
4
5
 
5
6
  class Evilution::Result::CoverageGapGrouper
@@ -7,10 +7,10 @@ require_relative "memory_stats"
7
7
  class Evilution::Result::MutationResult
8
8
  STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
9
9
 
10
- attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error
10
+ attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
11
11
 
12
12
  def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
13
- test_command: nil, memory: nil, error: nil)
13
+ test_command: nil, memory: nil, error: nil, neutral_reason: nil)
14
14
  raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
15
15
 
16
16
  @mutation = mutation
@@ -20,6 +20,7 @@ class Evilution::Result::MutationResult
20
20
  @test_command = test_command
21
21
  @memory = memory
22
22
  @error = error
23
+ @neutral_reason = neutral_reason
23
24
  freeze
24
25
  end
25
26
 
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # Why a mutation was recorded neutral.
6
+ #
7
+ # Neutral covers two unrelated situations, and they call for opposite responses:
8
+ # a spec file that was already red before any mutation ran, and a test process
9
+ # that died on infrastructure rather than on the mutation. Without the reason
10
+ # they are indistinguishable in a report, and a run can print full marks while
11
+ # the neutral bucket quietly holds what would otherwise be survivors
12
+ # (EV-5pob / GH #1606).
13
+ Evilution::Result::NeutralReason = Data.define(:kind, :detail) do
14
+ def self.baseline_failure(spec_file)
15
+ new(kind: :baseline_failure, detail: spec_file)
16
+ end
17
+
18
+ def self.infra_error(error_class)
19
+ new(kind: :infra_error, detail: error_class)
20
+ end
21
+
22
+ def to_s
23
+ detail ? "#{label} (#{detail})" : label
24
+ end
25
+
26
+ private
27
+
28
+ def label
29
+ case kind
30
+ when :baseline_failure then "baseline already failing"
31
+ when :infra_error then "infrastructure error"
32
+ else kind.to_s
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # What one subject — a single method — scored, alongside the counts behind it.
6
+ #
7
+ # The run's own score is computed per file, so a well-tested file that gains new
8
+ # untested methods still reports 100%: whatever the resolved spec reaches is all
9
+ # the number ever describes (EV-nlx1 / GH #1605). Per subject the picture
10
+ # separates: a method whose mutations were never reached scores nothing and says
11
+ # so, instead of disappearing into the file's total.
12
+ Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived) do
13
+ # Killed over what actually got a verdict, the same denominator the run uses.
14
+ def score
15
+ return 0.0 if verified.zero?
16
+
17
+ killed.to_f / verified
18
+ end
19
+
20
+ # Whether any mutation of this subject got a verdict at all.
21
+ def reached?
22
+ verified.positive?
23
+ end
24
+
25
+ def fully_verified?
26
+ reached? && killed == verified
27
+ end
28
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+ require_relative "subject_score"
5
+
6
+ # Groups mutation results by the subject they belong to and scores each one.
7
+ #
8
+ # Same denominator as the run's own score: statuses that carry no verdict
9
+ # (unresolved, neutral, equivalent, errored, unparseable) are left out of it, so
10
+ # a subject those account for entirely reports as unreached rather than as a
11
+ # perfect score over nothing.
12
+ class Evilution::Result::SubjectScorer
13
+ VERDICT_STATUSES = %i[killed survived timeout].freeze
14
+ private_constant :VERDICT_STATUSES
15
+
16
+ def call(results)
17
+ grouped = results.group_by { |result| [result.mutation.file_path, result.mutation.subject.name] }
18
+
19
+ grouped
20
+ .map { |(file_path, name), subject_results| score_for(name, file_path, subject_results) }
21
+ .sort_by { |score| [score.file_path, score.name] }
22
+ end
23
+
24
+ private
25
+
26
+ def score_for(name, file_path, results)
27
+ verdicts = results.count { |result| VERDICT_STATUSES.include?(result.status) }
28
+
29
+ Evilution::Result::SubjectScore.new(
30
+ name: name,
31
+ file_path: file_path,
32
+ total: results.length,
33
+ killed: results.count(&:killed?),
34
+ verified: verdicts,
35
+ survived: results.count(&:survived?)
36
+ )
37
+ end
38
+ end
@@ -2,19 +2,57 @@
2
2
 
3
3
  require_relative "../result"
4
4
  require_relative "coverage_gap_grouper"
5
+ require_relative "subject_scorer"
5
6
 
6
7
  class Evilution::Result::Summary
7
- attr_reader :results, :duration, :skipped, :disabled_mutations
8
+ attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
9
+ :target_file_count, :infra_retried
8
10
 
9
- def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [])
11
+ def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
12
+ unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
10
13
  @results = results
11
14
  @duration = duration
12
15
  @truncated = truncated
13
16
  @skipped = skipped
14
17
  @disabled_mutations = disabled_mutations
18
+ @unresolved_target_files = unresolved_target_files.freeze
19
+ @target_file_count = target_file_count
20
+ @infra_retried = infra_retried
15
21
  freeze
16
22
  end
17
23
 
24
+ # Neutral results gathered by the reason they were recorded, which is what the
25
+ # report needs to say which kind of neutral a reader is looking at
26
+ # (EV-5pob / GH #1606).
27
+ def neutral_results_by_reason
28
+ neutral_results.group_by(&:neutral_reason).to_a
29
+ end
30
+
31
+ # What each subject — each method — scored on its own. The run's score is
32
+ # computed per file, which says nothing about a method inside it that no
33
+ # example reaches (EV-nlx1 / GH #1605).
34
+ def subject_scores
35
+ Evilution::Result::SubjectScorer.new.call(results)
36
+ end
37
+
38
+ # The subjects the file-level score does not speak for: something survived, or
39
+ # nothing reached them at all.
40
+ def subjects_needing_attention
41
+ subject_scores.reject(&:fully_verified?)
42
+ end
43
+
44
+ # The same subjects, gathered under the file they live in, which is how the
45
+ # report lists them.
46
+ def subjects_needing_attention_by_file
47
+ subjects_needing_attention.group_by(&:file_path).values
48
+ end
49
+
50
+ # Files evilution was pointed at that resolved to no test file, so nothing
51
+ # about them was ever measured (EV-p4sm / GH #1603).
52
+ def unresolved_targets?
53
+ !unresolved_target_files.empty?
54
+ end
55
+
18
56
  def truncated?
19
57
  @truncated
20
58
  end
@@ -66,7 +104,11 @@ class Evilution::Result::Summary
66
104
  killed.to_f / denominator
67
105
  end
68
106
 
107
+ # A target file that was never tested fails the run on its own: the score
108
+ # only speaks for the files that did resolve to a spec.
69
109
  def success?(min_score: 1.0)
110
+ return false if unresolved_targets?
111
+
70
112
  score >= min_score
71
113
  end
72
114
 
@@ -60,18 +60,15 @@ class Evilution::Runner::BaselineRunner
60
60
  return nil unless config.baseline? && subjects.any?
61
61
 
62
62
  log_start
63
- baseline = Evilution::Baseline.new(
64
- timeout: config.timeout,
65
- test_files: config.spec_files.empty? ? nil : config.spec_files,
66
- **integration_class.baseline_options
67
- )
68
- result = baseline.call(subjects)
63
+ result = build_baseline.call(subjects)
69
64
  log_complete(result)
70
65
  result
71
66
  end
72
67
 
68
+ # The same selector the baseline resolves with, so a surviving mutation is
69
+ # matched against exactly the spec files the baseline ran.
73
70
  def neutralization_resolver
74
- integration_class.baseline_options[:spec_resolver] || Evilution::SpecResolver.new
71
+ config.spec_selector
75
72
  end
76
73
 
77
74
  def neutralization_fallback_dir
@@ -82,6 +79,19 @@ class Evilution::Runner::BaselineRunner
82
79
 
83
80
  attr_reader :config, :hooks
84
81
 
82
+ # Resolves through config.spec_selector -- the object `run` uses, so
83
+ # spec_mappings / spec_pattern apply -- and skips unresolved sources unless
84
+ # the run falls back to the full suite.
85
+ def build_baseline
86
+ Evilution::Baseline.new(
87
+ timeout: config.timeout,
88
+ test_files: config.spec_files.empty? ? nil : config.spec_files,
89
+ spec_selector: config.spec_selector,
90
+ fallback_to_full_suite: config.fallback_to_full_suite?,
91
+ **integration_class.baseline_options
92
+ )
93
+ end
94
+
85
95
  def build_example_filter
86
96
  return nil unless config.example_targeting?
87
97
 
@@ -34,7 +34,7 @@ class Evilution::Runner::Canary
34
34
  test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
35
35
  timeout: @config.timeout
36
36
  )
37
- raise Failed, failure_message(result.status) unless result.status == :survived
37
+ raise Failed, failure_message(result) unless result.status == :survived
38
38
 
39
39
  nil
40
40
  ensure
@@ -146,12 +146,59 @@ class Evilution::Runner::Canary
146
146
  @integration_class.new(test_files: [spec_path], hooks: @hooks)
147
147
  end
148
148
 
149
- def failure_message(status)
149
+ # When the child reported an error, that error IS the diagnosis -- naming it
150
+ # beats guessing. The speculative list stays only for the cases that carry no
151
+ # error at all (:killed, :timeout), where guesses are the only help there is.
152
+ # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
153
+ # this message used to drop, and none of the four guesses was the cause.
154
+ # EV-65nf / GH #1586.
155
+ def failure_message(result)
156
+ "#{failure_preamble(result.status)} #{diagnosis(result)} " \
157
+ "Re-run with --no-canary to bypass this check."
158
+ end
159
+
160
+ def failure_preamble(status)
150
161
  "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
151
162
  "mutation was scored #{status.inspect} instead of :survived. The mutation " \
152
- "pipeline is misreporting — every score this run would produce is unreliable. " \
153
- "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
163
+ "pipeline is misreporting — every score this run would produce is unreliable."
164
+ end
165
+
166
+ def diagnosis(result)
167
+ reported = reported_error(result)
168
+ return speculative_causes if reported.nil?
169
+
170
+ "The child reported: #{reported}"
171
+ end
172
+
173
+ def reported_error(result)
174
+ message = result.error_message
175
+ return nil if message.nil? || message.empty?
176
+
177
+ described = with_class_prefix(message, result.error_class)
178
+ frame = first_frame(result)
179
+ frame.nil? ? described : "#{described} (at #{frame})"
180
+ end
181
+
182
+ # MutationApplier already prefixes the class onto the message it packs
183
+ # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
184
+ # leave the class in its own field. Prefixing unconditionally would print
185
+ # "NameError: NameError: ..." for the former.
186
+ def with_class_prefix(message, klass)
187
+ return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
188
+
189
+ "#{klass}: #{message}"
190
+ end
191
+
192
+ def first_frame(result)
193
+ backtrace = result.error_backtrace
194
+ return nil if backtrace.nil? || backtrace.empty?
195
+
196
+ backtrace.first
197
+ end
198
+
199
+ def speculative_causes
200
+ "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
154
201
  "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
155
- "an isolation-mode defect. Re-run with --no-canary to bypass this check."
202
+ "an isolation-mode defect."
156
203
  end
157
204
  end
@@ -6,6 +6,7 @@ require_relative "../isolation/in_process"
6
6
  require_relative "../rails_detector"
7
7
  require_relative "../gem_detector"
8
8
  require_relative "../integration/loading/test_load_path"
9
+ require_relative "../rails_warmup"
9
10
 
10
11
  class Evilution::Runner::IsolationResolver
11
12
  PRELOAD_CANDIDATES = [
@@ -48,6 +49,7 @@ class Evilution::Runner::IsolationResolver
48
49
  prepare_load_path_for_preload(path)
49
50
  prepare_integration_for_preload
50
51
  require File.expand_path(path)
52
+ warm_up_rails
51
53
  rescue Evilution::ConfigError
52
54
  raise
53
55
  rescue ScriptError, StandardError => e
@@ -61,6 +63,21 @@ class Evilution::Runner::IsolationResolver
61
63
 
62
64
  attr_reader :config, :hooks
63
65
 
66
+ # Runs only after a preload (which is what loads Rails). This is primarily an
67
+ # optimisation for :fork isolation, where children inherit the warmed state.
68
+ def warm_up_rails
69
+ return unless config.warmup == :rails
70
+
71
+ application = ::Rails.application if defined?(::Rails) && ::Rails.respond_to?(:application)
72
+ unless application
73
+ warn_message = "[evilution] warmup: rails requested but Rails is not loaded (or has no application) after preload; skipping"
74
+ Evilution::Diagnostic.warn(warn_message)
75
+ return
76
+ end
77
+
78
+ Evilution::RailsWarmup.new(application).call
79
+ end
80
+
64
81
  # Under :fork, allow preloading — caller resolves whether a path exists (an
65
82
  # explicit --preload / preload: value, or an auto-detected rails_helper) and
66
83
  # bails early when none does. Under :in_process, only allow preloading when
@@ -131,7 +148,11 @@ class Evilution::Runner::IsolationResolver
131
148
  # have both (a bare `require "support/foo"` must still resolve from spec/).
132
149
  # - minitest/test-unit: a test/test_helper.rb doing a non-relative
133
150
  # `require "support/..."` needs test/ on $LOAD_PATH. Route through
134
- # TestLoadPath -- the same policy the per-mutation test load uses -- so preload and mutation paths agree.
151
+ # TestLoadPath, the policy the per-mutation test load uses, with one
152
+ # deliberate difference: outside_root_dirs: false. A resolved test file
153
+ # outside test/ and spec/ keeps its own directory, but a preload there
154
+ # (config/evilution_preload.rb) must not, or config/ lands ahead of the
155
+ # gems and config/puma.rb shadows the puma gem (GH #1597).
135
156
  def prepare_load_path_for_preload(preload_path)
136
157
  if config.integration == :rspec
137
158
  prepare_rspec_preload_load_path
@@ -148,7 +169,7 @@ class Evilution::Runner::IsolationResolver
148
169
 
149
170
  def prepare_test_preload_load_path(preload_path)
150
171
  base = detected_rails_root || Evilution.project_base_dir
151
- Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base)
172
+ Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base, outside_root_dirs: false)
152
173
  end
153
174
 
154
175
  def resolve_spec_dir
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+ require_relative "neutralizer/infra_error"
5
+
6
+ # Re-runs, one at a time, the mutations a parallel pass could not judge because
7
+ # the test process crashed on infrastructure — a database lock, a statement
8
+ # timeout — rather than on the mutation itself.
9
+ #
10
+ # Those crashes are demoted to `:neutral` so they do not inflate the kill count
11
+ # (EV-toid / GH #814), which leaves the neutral bucket swinging with `--jobs`
12
+ # on identical input: the contention that causes them only exists while several
13
+ # workers run at once. Re-running them after the pool is done removes the
14
+ # contention, so the verdict matches what a serial run would have produced
15
+ # (EV-j0bv / GH #1607).
16
+ #
17
+ # Only mutations neutralised by an infra crash are re-run. A neutral from a
18
+ # failing baseline is a real statement about the spec, and re-running it would
19
+ # say the same thing again.
20
+ class Evilution::Runner::MutationExecutor::InfraRetry
21
+ attr_reader :retried_count
22
+
23
+ def initialize(runner:, pipeline:)
24
+ @runner = runner
25
+ @pipeline = pipeline
26
+ @retried_count = 0
27
+ end
28
+
29
+ def call(results, baseline_result:, integration:)
30
+ @retried_count = 0
31
+
32
+ results.map do |result|
33
+ next result unless retryable?(result)
34
+
35
+ @retried_count += 1
36
+ rerun(result, baseline_result: baseline_result, integration: integration)
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def retryable?(result)
43
+ Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_neutral?(result)
44
+ end
45
+
46
+ # The parallel pass keeps the sources of a retryable mutation alive so it can
47
+ # be applied again; they are released here, once it has been.
48
+ def rerun(result, baseline_result:, integration:)
49
+ mutation = result.mutation
50
+ rerun_result = @runner.call(mutation, integration: integration)
51
+ mutation.strip_sources!
52
+ @pipeline.call(rerun_result, baseline_result: baseline_result)
53
+ end
54
+ end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative "../neutralizer"
4
4
  require_relative "../../../result/mutation_result"
5
+ require_relative "../../../result/neutral_reason"
5
6
 
6
7
  class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
7
8
  def initialize(config:, spec_resolver:, fallback_dir:)
@@ -13,27 +14,39 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
13
14
  def call(result, baseline_result:)
14
15
  return result unless result.survived? && baseline_result && baseline_result.failed?
15
16
 
16
- if @config.spec_files.any?
17
- should_neutralize = true
18
- else
19
- spec_file = @spec_resolver.call(result.mutation.file_path) || @fallback_dir
20
- should_neutralize = baseline_result.failed_spec_files.include?(spec_file)
21
- end
22
- return result unless should_neutralize
17
+ return neutralize(result, nil) if @config.spec_files.any?
23
18
 
24
- neutralize(result)
19
+ failed = baseline_result.failed_spec_files
20
+ spec_file = covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
21
+ return result unless spec_file
22
+
23
+ neutralize(result, spec_file)
25
24
  end
26
25
 
27
26
  private
28
27
 
29
- def neutralize(result)
28
+ # The spec files the baseline ran for this source: every file the resolver
29
+ # returns (a spec_selector may map one source to several), or the fallback
30
+ # directory when nothing resolves -- but only if the run falls back to the
31
+ # full suite; otherwise the baseline never ran it.
32
+ def covering_specs(file_path)
33
+ specs = Array(@spec_resolver.call(file_path))
34
+ return specs unless specs.empty?
35
+
36
+ @config.fallback_to_full_suite? ? [@fallback_dir] : []
37
+ end
38
+
39
+ # The spec that was already red is the fact a reader needs: it says what to
40
+ # fix before the run can say anything about these mutations.
41
+ def neutralize(result, spec_file)
30
42
  Evilution::Result::MutationResult.new(
31
43
  mutation: result.mutation,
32
44
  status: :neutral,
33
45
  duration: result.duration,
34
46
  test_command: result.test_command,
35
47
  memory: result.memory,
36
- error: result.error
48
+ error: result.error,
49
+ neutral_reason: Evilution::Result::NeutralReason.baseline_failure(spec_file)
37
50
  )
38
51
  end
39
52
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative "../neutralizer"
4
4
  require_relative "../../../result/mutation_result"
5
+ require_relative "../../../result/neutral_reason"
5
6
 
6
7
  # Reclassify results as :neutral when the failure was caused by test
7
8
  # infrastructure rather than by the mutation. Two independent paths:
@@ -33,6 +34,21 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
33
34
  ].freeze
34
35
  private_constant :INFRA_ERROR_CLASSES, :INFRA_BACKTRACE_PATHS, :INFRA_CRASH_CLASSES
35
36
 
37
+ # Whether a crash class is one of the infrastructure failures that say
38
+ # nothing about the mutation. The parallel strategy asks this before the
39
+ # neutralisation pipeline has run, while the result is still a `:killed`
40
+ # crash (EV-j0bv / GH #1607).
41
+ def self.infra_crash_class?(error_class)
42
+ INFRA_CRASH_CLASSES.include?(error_class)
43
+ end
44
+
45
+ # Whether this result is a kill that was demoted because the test process
46
+ # crashed on infrastructure rather than on the mutation. Such a mutation got
47
+ # no verdict, so a parallel run can re-run it once the contention is over.
48
+ def self.infra_neutral?(result)
49
+ result.neutral? && infra_crash_class?(result.error_class)
50
+ end
51
+
36
52
  def call(result, **_ctx)
37
53
  return neutralize(result) if infra_crash?(result)
38
54
  return result unless result.error?
@@ -62,7 +78,8 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
62
78
  duration: result.duration,
63
79
  test_command: result.test_command,
64
80
  memory: result.memory,
65
- error: result.error
81
+ error: result.error,
82
+ neutral_reason: Evilution::Result::NeutralReason.infra_error(result.error_class)
66
83
  )
67
84
  end
68
85
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require_relative "../mutation_executor"
4
4
  require_relative "../../result/mutation_result"
5
+ require_relative "neutralizer/infra_error"
5
6
 
6
7
  class Evilution::Runner::MutationExecutor::ResultCache
7
8
  CACHEABLE_STATUSES = %i[killed timeout].freeze
@@ -32,6 +33,7 @@ class Evilution::Runner::MutationExecutor::ResultCache
32
33
  def store(mutation, result)
33
34
  return unless @backend
34
35
  return unless result.killed? || result.timeout?
36
+ return if infra_crash?(result)
35
37
 
36
38
  @backend.store(mutation,
37
39
  status: result.status,
@@ -40,6 +42,15 @@ class Evilution::Runner::MutationExecutor::ResultCache
40
42
  test_command: result.test_command)
41
43
  end
42
44
 
45
+ # A kill that came from an infrastructure crash is not a verdict about the
46
+ # mutation, and the cache keeps no error class to tell them apart later. It
47
+ # is left out so the next fetch does not hand back a plain `:killed`, which
48
+ # would defeat both the demotion to neutral and the serial retry
49
+ # (EV-j0bv / GH #1607).
50
+ def infra_crash?(result)
51
+ Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
52
+ end
53
+
43
54
  def partition(batch, packer:)
44
55
  uncached_indices = []
45
56
  cached_results = {}
@@ -65,10 +65,28 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
65
65
  compact_results = merge(batch, partition.uncached_indices, partition.cached_results, worker_results)
66
66
  batch_results = rebuild_results(batch, compact_results)
67
67
  cache_results(batch_results, partition.uncached_indices)
68
- batch.each(&:strip_sources!)
68
+ release_sources(batch, batch_results)
69
69
  batch_results
70
70
  end
71
71
 
72
+ # Sources are dropped as soon as a mutation has been judged, to keep a long
73
+ # run's memory flat. A mutation the workers could not judge because of
74
+ # infrastructure contention is re-run serially once the pool is done
75
+ # (InfraRetry), which needs its sources, so it keeps them until then. The
76
+ # neutralisation pipeline has not run yet here, so the crash is still a kill.
77
+ def release_sources(batch, batch_results)
78
+ batch.each_with_index do |mutation, index|
79
+ result = batch_results[index]
80
+ next if result && infra_crash?(result)
81
+
82
+ mutation.strip_sources!
83
+ end
84
+ end
85
+
86
+ def infra_crash?(result)
87
+ Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
88
+ end
89
+
72
90
  def rebuild_results(batch, compact_results)
73
91
  batch.zip(compact_results).map { |m, h| @packer.rebuild(m, h) }
74
92
  end