evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # Where a run's time went, apart from what each mutation measured itself:
6
+ #
7
+ # - phases: seconds per phase of the run, in the order they ran
8
+ # - baseline_spec_files: seconds the baseline spent on each spec file
9
+ # - workers: one Parallel::WorkQueue::WorkerStat per worker process
10
+ # of a parallel run; empty for a sequential one
11
+ class Evilution::Result::RunTimings
12
+ attr_reader :phases, :baseline_spec_files, :workers
13
+
14
+ def initialize(phases: {}, baseline_spec_files: {}, workers: [])
15
+ @phases = phases.freeze
16
+ @baseline_spec_files = baseline_spec_files.freeze
17
+ @workers = workers.freeze
18
+ freeze
19
+ end
20
+ end
@@ -9,7 +9,14 @@ require_relative "../result"
9
9
  # the number ever describes (EV-nlx1 / GH #1605). Per subject the picture
10
10
  # separates: a method whose mutations were never reached scores nothing and says
11
11
  # so, instead of disappearing into the file's total.
12
- Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived) do
12
+ #
13
+ # uncovered: the mutations no test executes, counted in verified as the
14
+ # survivors they would have been.
15
+ Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived, :uncovered) do
16
+ def initialize(name:, file_path:, total:, killed:, verified:, survived:, uncovered: 0)
17
+ super
18
+ end
19
+
13
20
  # Killed over what actually got a verdict, the same denominator the run uses.
14
21
  def score
15
22
  return 0.0 if verified.zero?
@@ -8,9 +8,10 @@ require_relative "subject_score"
8
8
  # Same denominator as the run's own score: statuses that carry no verdict
9
9
  # (unresolved, neutral, equivalent, errored, unparseable) are left out of it, so
10
10
  # a subject those account for entirely reports as unreached rather than as a
11
- # perfect score over nothing.
11
+ # perfect score over nothing. An uncovered mutation stays in, as the survivor
12
+ # it would have been.
12
13
  class Evilution::Result::SubjectScorer
13
- VERDICT_STATUSES = %i[killed survived timeout].freeze
14
+ VERDICT_STATUSES = %i[killed survived timeout uncovered].freeze
14
15
  private_constant :VERDICT_STATUSES
15
16
 
16
17
  def call(results)
@@ -32,7 +33,8 @@ class Evilution::Result::SubjectScorer
32
33
  total: results.length,
33
34
  killed: results.count(&:killed?),
34
35
  verified: verdicts,
35
- survived: results.count(&:survived?)
36
+ survived: results.count(&:survived?),
37
+ uncovered: results.count(&:uncovered?)
36
38
  )
37
39
  end
38
40
  end
@@ -4,27 +4,47 @@ require_relative "../result"
4
4
  require_relative "coverage_gap_grouper"
5
5
  require_relative "subject_scorer"
6
6
  require_relative "baseline_neutralization"
7
+ require_relative "run_timings"
8
+ require_relative "mutation_timings"
7
9
 
8
10
  class Evilution::Result::Summary
9
- attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
10
- :target_file_count, :infra_retried, :uncovered_code, :baseline_failures
11
+ attr_reader :results, :duration, :skipped, :skipped_sites, :disabled_mutations, :unresolved_target_files,
12
+ :target_file_count, :infra_retried, :uncovered_code, :baseline_failures, :timings
11
13
 
12
- def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
14
+ # skipped: every mutation left out during generation (ignore_patterns,
15
+ # skip_sites, disable comments, Sorbet sigs); skipped_sites: the share of
16
+ # each skip_sites preset, e.g. { logging: 3 }.
17
+ def initialize(results:, duration: 0.0, truncated: false, skipped: 0, skipped_sites: {}, disabled_mutations: [],
13
18
  unresolved_target_files: [], target_file_count: nil, infra_retried: 0, uncovered_code: [],
14
- baseline_failures: [])
19
+ baseline_failures: [], timings: Evilution::Result::RunTimings.new)
15
20
  @results = results
16
21
  @duration = duration
17
22
  @truncated = truncated
18
23
  @skipped = skipped
24
+ @skipped_sites = skipped_sites.dup.freeze
19
25
  @disabled_mutations = disabled_mutations
20
26
  @unresolved_target_files = unresolved_target_files.freeze
21
27
  @target_file_count = target_file_count
22
28
  @infra_retried = infra_retried
23
29
  @uncovered_code = uncovered_code.freeze
24
30
  @baseline_failures = baseline_failures.freeze
31
+ @timings = timings
25
32
  freeze
26
33
  end
27
34
 
35
+ # Each mutation phase summed over the mutations that measured it, in
36
+ # seconds of process time: in a parallel run they add up across workers.
37
+ def mutation_timing_totals
38
+ totals = Evilution::Result::MutationTimings::PHASES.to_h { |phase| [phase, 0.0] }
39
+ results.each do |result|
40
+ timings = result.timings
41
+ next unless timings
42
+
43
+ timings.to_h.each { |phase, seconds| totals[phase] += seconds }
44
+ end
45
+ totals
46
+ end
47
+
28
48
  # The kills not counted because only examples already failing in the
29
49
  # baseline failed, gathered under the spec file that was red and paired with
30
50
  # why it was. They are out of the score, so a red spec file would read as
@@ -119,6 +139,18 @@ class Evilution::Result::Summary
119
139
  results.count(&:unparseable?)
120
140
  end
121
141
 
142
+ def uncovered
143
+ results.count(&:uncovered?)
144
+ end
145
+
146
+ # The survivors a probe-first sentinel implied: counted in survived, but
147
+ # they did not run.
148
+ def implied
149
+ results.count(&:implied_by)
150
+ end
151
+
152
+ # An uncovered mutation stays in: it is untested code, scored as the
153
+ # survivor running it would have produced.
122
154
  def score_denominator
123
155
  total - errors - neutral - equivalent - unresolved - unparseable
124
156
  end
@@ -162,7 +194,19 @@ class Evilution::Result::Summary
162
194
  results.select(&:unparseable?)
163
195
  end
164
196
 
197
+ def uncovered_results
198
+ results.select(&:uncovered?)
199
+ end
200
+
201
+ # The survivors, and the mutations no test executes: survivors that were
202
+ # not run.
165
203
  def coverage_gaps
204
+ Evilution::Result::CoverageGapGrouper.new.call(survived_results + uncovered_results)
205
+ end
206
+
207
+ # The survivors' gaps alone, for a report that lists the uncovered
208
+ # mutations apart.
209
+ def survived_gaps
166
210
  Evilution::Result::CoverageGapGrouper.new.call(survived_results)
167
211
  end
168
212
 
@@ -183,14 +227,7 @@ class Evilution::Result::Summary
183
227
  end
184
228
 
185
229
  def peak_memory_mb
186
- max_rss = nil
187
- results.each do |result|
188
- kb = result.child_rss_kb
189
- next unless kb
190
-
191
- max_rss = kb if max_rss.nil? || kb > max_rss
192
- end
193
-
230
+ max_rss = results.filter_map(&:child_rss_kb).max
194
231
  max_rss && (max_rss / 1024.0)
195
232
  end
196
233
 
@@ -8,6 +8,7 @@ require_relative "../integration/minitest"
8
8
  require_relative "../coverage_example_filter"
9
9
  require_relative "../coverage/map_store"
10
10
  require_relative "../coverage/map_builder"
11
+ require_relative "../cache/run_fingerprint"
11
12
  require_relative "../integration/test_unit"
12
13
  require_relative "../example_filter"
13
14
  require_relative "../spec_ast_cache"
@@ -40,7 +41,7 @@ class Evilution::Runner::BaselineRunner
40
41
  def build_integration(baseline_result = nil)
41
42
  klass = integration_class
42
43
  kwargs = base_integration_kwargs(baseline_result)
43
- kwargs.merge!(rspec_integration_kwargs) if klass == Evilution::Integration::RSpec
44
+ kwargs.merge!(rspec_integration_kwargs(baseline_result)) if klass == Evilution::Integration::RSpec
44
45
  klass.new(**kwargs)
45
46
  end
46
47
 
@@ -50,14 +51,19 @@ class Evilution::Runner::BaselineRunner
50
51
  test_files: config.spec_files.empty? ? nil : config.spec_files,
51
52
  hooks: hooks,
52
53
  fallback_to_full_suite: config.fallback_to_full_suite?,
53
- spec_selector: config.spec_selector
54
+ spec_selector: config.spec_selector,
55
+ stop_on_kill: config.stop_on_kill?
54
56
  }
55
57
  end
56
58
 
57
- def rspec_integration_kwargs
59
+ # example_durations: what each example cost in the baseline, so a run that
60
+ # stops at its first kill can try the cheapest first, when killer_first asks.
61
+ def rspec_integration_kwargs(baseline_result)
58
62
  {
59
63
  related_specs_heuristic: config.related_specs_heuristic?,
60
- example_filter: build_example_filter
64
+ example_filter: build_example_filter,
65
+ killer_first: config.killer_first?,
66
+ example_durations: baseline_result ? baseline_result.example_durations : {}
61
67
  }
62
68
  end
63
69
 
@@ -135,8 +141,12 @@ class Evilution::Runner::BaselineRunner
135
141
  specs = resolved_spec_files
136
142
  return nil if specs.empty? # nothing resolves -> no coverage to capture, lexical handles it
137
143
 
144
+ load_or_build_coverage_map(targets, specs)
145
+ end
146
+
147
+ def load_or_build_coverage_map(targets, specs)
138
148
  store = Evilution::Coverage::MapStore.new
139
- cache_inputs = targets + specs
149
+ cache_inputs = (targets + specs + coverage_helper_files + spec_tree_files(specs)).uniq
140
150
  return store.load(targets) if store.stale_files(cache_inputs).empty?
141
151
 
142
152
  map = Evilution::Coverage::MapBuilder.new(spec_files: specs, target_files: targets).call
@@ -144,6 +154,38 @@ class Evilution::Runner::BaselineRunner
144
154
  map
145
155
  end
146
156
 
157
+ # What every spec file loads (the same helpers the incremental cache keys
158
+ # on): a support file that starts or stops calling the code changes which
159
+ # lines run, and the map says which lines no example executes.
160
+ def coverage_helper_files
161
+ paths = Evilution::Cache::RunFingerprint::HELPERS + Array(config.preload).map(&:to_s)
162
+ paths.flat_map { |path| helper_files_at(File.expand_path(path, Evilution::PROJECT_ROOT)) }.uniq
163
+ end
164
+
165
+ # Every file under the directories the spec files sit in (spec/, test/):
166
+ # shared examples, factories and fixtures data can change which lines run,
167
+ # and not all of them are loaded as Ruby files the build would record.
168
+ def spec_tree_files(specs)
169
+ roots = specs.filter_map { |spec| spec_root(spec) }.uniq
170
+ roots.flat_map { |root| Dir.glob(File.join(root, "**", "*")).select { |path| File.file?(path) } }
171
+ end
172
+
173
+ # The top directory of the project a spec file sits under; nil for one
174
+ # outside the project or at its top.
175
+ def spec_root(spec)
176
+ relative = spec.delete_prefix("#{Evilution::PROJECT_ROOT}/")
177
+ top, rest = relative.split("/", 2)
178
+ return nil if relative == spec || rest.nil?
179
+
180
+ File.join(Evilution::PROJECT_ROOT, top)
181
+ end
182
+
183
+ def helper_files_at(path)
184
+ return Dir.glob(File.join(path, "**", "*.rb")) if File.directory?(path)
185
+
186
+ File.file?(path) ? [path] : []
187
+ end
188
+
147
189
  # The RESOLVED spec files for the target sources -- the same lib-mirrored specs
148
190
  # full-file targeting already runs cleanly -- NOT the whole suite. EV-7uui:
149
191
  # capturing and replaying coverage only within these guarantees the covering
@@ -60,19 +60,6 @@ class Evilution::Runner::Diagnostics
60
60
  end
61
61
  end
62
62
 
63
- def aggregate_worker_stats(stats)
64
- return stats if stats.empty?
65
-
66
- stats.group_by(&:pid).map do |pid, entries|
67
- Evilution::Parallel::WorkQueue::WorkerStat.new(
68
- pid,
69
- entries.sum(&:items_completed),
70
- entries.sum(&:busy_time),
71
- entries.sum(&:wall_time)
72
- )
73
- end
74
- end
75
-
76
63
  private
77
64
 
78
65
  attr_reader :config, :stderr
@@ -39,15 +39,20 @@ class Evilution::Runner::IsolationResolver
39
39
  @rails_root_detected = !detected_rails_root.nil?
40
40
  end
41
41
 
42
- def perform_preload
43
- return if config.preload == false
44
- return unless should_preload?
42
+ # Whether the tests run in children forked from this process.
43
+ def fork_isolation?
44
+ resolve_isolation == :fork
45
+ end
45
46
 
46
- path = resolve_preload_path
47
+ # Yields just before the preload file is required, and only when there is
48
+ # one, so a caller can count what the preload runs.
49
+ def perform_preload
50
+ path = preload_path
47
51
  return unless path
48
52
 
49
53
  prepare_load_path_for_preload(path)
50
54
  prepare_integration_for_preload
55
+ yield if block_given?
51
56
  require File.expand_path(path)
52
57
  warm_up_rails
53
58
  rescue Evilution::ConfigError
@@ -63,6 +68,13 @@ class Evilution::Runner::IsolationResolver
63
68
 
64
69
  attr_reader :config, :hooks
65
70
 
71
+ def preload_path
72
+ return nil if config.preload == false
73
+ return nil unless should_preload?
74
+
75
+ resolve_preload_path
76
+ end
77
+
66
78
  # Runs only after a preload (which is what loads Rails). This is primarily an
67
79
  # optimisation for :fork isolation, where children inherit the warmed state.
68
80
  def warm_up_rails
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+
5
+ # One mutation's test run under the isolator, within its time limit.
6
+ #
7
+ # A run whose tests the integration tried in an order of its own (see
8
+ # Integration::Base#reorders?) and that timed out is run again in the defined
9
+ # order, and that run decides. The reordering may have started with the very
10
+ # test the mutant hangs, ahead of one that would have failed on it at once, so
11
+ # its timeout says nothing about what the defined order would have found.
12
+ class Evilution::Runner::MutationExecutor::IsolatedRun
13
+ def initialize(isolator)
14
+ @isolator = isolator
15
+ end
16
+
17
+ def call(mutation, integration, limit)
18
+ reordered = reorders?(integration)
19
+ result = isolate(mutation, integration, limit)
20
+ result = integration.in_defined_order { isolate(mutation, integration, limit) } if reordered && result.timeout?
21
+ result.timeout? ? result.with_time_limit(limit) : result
22
+ end
23
+
24
+ private
25
+
26
+ # Asked before the run: the integration's answer is about the run to come.
27
+ def reorders?(integration)
28
+ integration.respond_to?(:reorders?) && integration.reorders?
29
+ end
30
+
31
+ def isolate(mutation, integration, limit)
32
+ @isolator.call(mutation: mutation, test_command: ->(m) { integration.call(m) }, timeout: limit)
33
+ end
34
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+
5
+ # The tests that killed mutations earlier in this run, kept in the parent so
6
+ # each mutation can be sent out with the tests most likely to kill it: those
7
+ # that killed a mutation on its own line, then those that killed one elsewhere
8
+ # in its subject. The latest killer of each comes first.
9
+ #
10
+ # Only the order of a mutation's tests comes of it, never which tests run, so
11
+ # a killer that has gone stale costs one test run at the most.
12
+ class Evilution::Runner::MutationExecutor::KillHistory
13
+ LIMIT = 16
14
+ EMPTY = [].freeze
15
+ private_constant :EMPTY
16
+
17
+ # Remembers nothing: for a run, or a test, that does without.
18
+ class None
19
+ def record(_mutation, status:, killing_test:); end
20
+
21
+ def killers_for(_mutation)
22
+ EMPTY
23
+ end
24
+ end
25
+ private_constant :None
26
+
27
+ NONE = None.new.freeze
28
+
29
+ def initialize
30
+ @by_line = {}
31
+ @by_subject = {}
32
+ end
33
+
34
+ def record(mutation, status:, killing_test:)
35
+ return unless status == :killed && killing_test
36
+
37
+ remember(@by_line, line_key(mutation), killing_test)
38
+ subject = subject_key(mutation)
39
+ remember(@by_subject, subject, killing_test) if subject
40
+ end
41
+
42
+ def killers_for(mutation)
43
+ line = @by_line.fetch(line_key(mutation), EMPTY)
44
+ subject = subject_key(mutation)
45
+ nearby = subject ? @by_subject.fetch(subject, EMPTY) : EMPTY
46
+ (line + nearby).uniq.first(LIMIT)
47
+ end
48
+
49
+ private
50
+
51
+ def remember(index, key, test)
52
+ killers = (index[key] ||= [])
53
+ killers.delete(test)
54
+ killers.unshift(test)
55
+ killers.pop while killers.length > LIMIT
56
+ end
57
+
58
+ def line_key(mutation)
59
+ [mutation.file_path, mutation.line]
60
+ end
61
+
62
+ def subject_key(mutation)
63
+ subject = mutation.subject
64
+ subject ? [mutation.file_path, subject.name] : nil
65
+ end
66
+ end
@@ -2,29 +2,48 @@
2
2
 
3
3
  require_relative "../mutation_executor"
4
4
  require_relative "../../result/mutation_result"
5
+ require_relative "runless_verdict"
6
+ require_relative "kill_history"
7
+ require_relative "timed_run"
5
8
 
6
9
  class Evilution::Runner::MutationExecutor::MutationRunner
7
- def initialize(config:, cache:, isolator:)
8
- @config = config
10
+ # timeout_policy: answers the time limit for each mutation's test run.
11
+ # runless: the verdict a mutation gets without a run, when it has one.
12
+ # kill_history: the tests that killed earlier mutations of the run (see
13
+ # KillHistory), handed to the integration so it can try them first.
14
+ def initialize(cache:, isolator:, timeout_policy:,
15
+ runless: Evilution::Runner::MutationExecutor::RunlessVerdict.new,
16
+ kill_history: Evilution::Runner::MutationExecutor::KillHistory::NONE)
9
17
  @cache = cache
10
- @isolator = isolator
18
+ @timeout_policy = timeout_policy
19
+ @runless = runless
20
+ @kill_history = kill_history
21
+ @timed_run = Evilution::Runner::MutationExecutor::TimedRun.new(isolator: isolator, timeout_policy: timeout_policy)
11
22
  end
12
23
 
13
24
  def call(mutation, integration:)
14
- return unparseable_result(mutation) if mutation.unparseable?
25
+ runless = @runless.call(mutation)
26
+ return runless if runless
15
27
 
16
- cached = @cache.fetch(mutation)
17
- return cached if cached
28
+ cached = @cache.fetch(mutation, time_limit: @timeout_policy.final_limit(mutation))
29
+ return remembered(mutation, cached) if cached
18
30
 
19
- test_command = ->(m) { integration.call(m) }
20
- result = @isolator.call(mutation: mutation, test_command: test_command, timeout: @config.timeout)
21
- @cache.store(mutation, result)
22
- result
31
+ remembered(mutation, run(mutation, integration))
23
32
  end
24
33
 
25
34
  private
26
35
 
27
- def unparseable_result(mutation)
28
- Evilution::Result::MutationResult.new(mutation: mutation, status: :unparseable)
36
+ # The integration is told before the isolator runs, so a forked child
37
+ # inherits what it was told.
38
+ def run(mutation, integration)
39
+ integration.prioritize(@kill_history.killers_for(mutation)) if integration.respond_to?(:prioritize)
40
+ result = @timed_run.call(mutation, integration)
41
+ @cache.store(mutation, result)
42
+ result
43
+ end
44
+
45
+ def remembered(mutation, result)
46
+ @kill_history.record(mutation, status: result.status, killing_test: result.killing_test)
47
+ result
29
48
  end
30
49
  end
@@ -60,6 +60,7 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
60
60
  test_command: result.test_command,
61
61
  memory: result.memory,
62
62
  error: result.error,
63
+ timings: result.timings,
63
64
  neutral_reason: Evilution::Result::NeutralReason.baseline_failure(spec_file)
64
65
  )
65
66
  end
@@ -3,6 +3,7 @@
3
3
  require_relative "../neutralizer"
4
4
  require_relative "../../../result/mutation_result"
5
5
  require_relative "../../../result/neutral_reason"
6
+ require_relative "../../../infra_crash"
6
7
 
7
8
  # Reclassify results as :neutral when the failure was caused by test
8
9
  # infrastructure rather than by the mutation. Two independent paths:
@@ -15,7 +16,7 @@ require_relative "../../../result/neutral_reason"
15
16
  # mutation NameError/LoadError as :neutral.
16
17
  #
17
18
  # 2) :killed from a CrashDetector test_crashed whose sole crash class is in
18
- # INFRA_CRASH_CLASSES (ActiveRecord::StatementTimeout, Timeout::Error,
19
+ # Evilution::InfraCrash::CLASSES (ActiveRecord::StatementTimeout, Timeout::Error,
19
20
  # etc.). These surface under parallel workers sharing a DB file or on a
20
21
  # slow CI; fork.rb initially reports them as :killed, and without this
21
22
  # demotion the kill count inflates with infra noise. No backtrace check:
@@ -24,29 +25,13 @@ require_relative "../../../result/neutral_reason"
24
25
  class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
25
26
  INFRA_ERROR_CLASSES = %w[LoadError NameError].freeze
26
27
  INFRA_BACKTRACE_PATHS = %r{(?:^|/)(?:spec_helper\.rb|rails_helper\.rb|spec/support/)}
27
- INFRA_CRASH_CLASSES = %w[
28
- Timeout::Error
29
- ActiveRecord::StatementTimeout
30
- ActiveRecord::Deadlocked
31
- ActiveRecord::ConnectionTimeoutError
32
- ActiveRecord::LockWaitTimeout
33
- SQLite3::BusyException
34
- ].freeze
35
- private_constant :INFRA_ERROR_CLASSES, :INFRA_BACKTRACE_PATHS, :INFRA_CRASH_CLASSES
36
-
37
- # Whether a crash class is one of the infrastructure failures that say
38
- # nothing about the mutation. The parallel strategy asks this before the
39
- # neutralisation pipeline has run, while the result is still a `:killed`
40
- # crash (EV-j0bv / GH #1607).
41
- def self.infra_crash_class?(error_class)
42
- INFRA_CRASH_CLASSES.include?(error_class)
43
- end
28
+ private_constant :INFRA_ERROR_CLASSES, :INFRA_BACKTRACE_PATHS
44
29
 
45
30
  # Whether this result is a kill that was demoted because the test process
46
31
  # crashed on infrastructure rather than on the mutation. Such a mutation got
47
32
  # no verdict, so a parallel run can re-run it once the contention is over.
48
33
  def self.infra_neutral?(result)
49
- result.neutral? && infra_crash_class?(result.error_class)
34
+ result.neutral? && Evilution::InfraCrash.class_name?(result.error_class)
50
35
  end
51
36
 
52
37
  def call(result, **_ctx)
@@ -61,7 +46,7 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
61
46
  private
62
47
 
63
48
  def infra_crash?(result)
64
- result.killed? && INFRA_CRASH_CLASSES.include?(result.error_class)
49
+ result.killed? && Evilution::InfraCrash.class_name?(result.error_class)
65
50
  end
66
51
 
67
52
  def infra_origin?(backtrace)
@@ -79,6 +64,7 @@ class Evilution::Runner::MutationExecutor::Neutralizer::InfraError
79
64
  test_command: result.test_command,
80
65
  memory: result.memory,
81
66
  error: result.error,
67
+ timings: result.timings,
82
68
  neutral_reason: Evilution::Result::NeutralReason.infra_error(result.error_class)
83
69
  )
84
70
  end
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prism"
4
+ require_relative "../mutation_executor"
5
+
6
+ # Which subjects no selected test exercises, learnt from one sentinel mutation
7
+ # per subject that runs before the others: the method's whole body replaced by
8
+ # a bare `raise`. When even that survives, no test calls the method on a path
9
+ # it checks, and the subject's other mutations would almost all survive too;
10
+ # they are reported as survivors implied by the sentinel, without a run.
11
+ #
12
+ # The sentinel is the body raise of the subject's own method, the one reported
13
+ # at the subject's line; a method nested inside has its own line and is just
14
+ # another mutation of the subject. A subject without a sentinel (an empty
15
+ # body, a body that already only raises, a subject that is not a method) is
16
+ # never skipped.
17
+ #
18
+ # Nor is a method with a rescue or ensure clause of its own: the body raise
19
+ # replaces only the statements before it and keeps the clause, which can
20
+ # catch the raise (`rescue StandardError`) or return over it (`ensure;
21
+ # return x`), so the method returns normally and the sentinel survives while
22
+ # the tests check the method's result.
23
+ class Evilution::Runner::MutationExecutor::ProbeFirst
24
+ SENTINEL_OPERATOR = "method_body_to_raise"
25
+ # A sentinel no test executes says the same as one that ran and survived.
26
+ UNEXERCISED_STATUSES = %i[survived uncovered].freeze
27
+ private_constant :UNEXERCISED_STATUSES
28
+
29
+ # Probes nothing: for a run without probe-first.
30
+ class None
31
+ def sentinel?(_mutation)
32
+ false
33
+ end
34
+
35
+ def record(_result); end
36
+
37
+ def implied_by(_mutation)
38
+ nil
39
+ end
40
+ end
41
+ private_constant :None
42
+
43
+ NONE = None.new.freeze
44
+
45
+ def initialize
46
+ @unexercised = Set.new
47
+ @probeable = {}
48
+ end
49
+
50
+ def sentinel?(mutation)
51
+ subject = mutation.subject
52
+ return false if subject.nil?
53
+ return false unless mutation.operator_name == SENTINEL_OPERATOR && mutation.line == subject.line_number
54
+
55
+ probeable?(subject)
56
+ end
57
+
58
+ # The sentinels go first, so every one of them has a verdict before any
59
+ # other mutation is looked at.
60
+ def split(mutations)
61
+ mutations.partition { |mutation| sentinel?(mutation) }
62
+ end
63
+
64
+ def record(result)
65
+ mutation = result.mutation
66
+ return unless sentinel?(mutation)
67
+ return unless UNEXERCISED_STATUSES.include?(result.status)
68
+
69
+ @unexercised << key(mutation)
70
+ end
71
+
72
+ # The operator of the sentinel that implies the mutation survives, or nil
73
+ # when the mutation has to run.
74
+ def implied_by(mutation)
75
+ return nil if mutation.subject.nil? || sentinel?(mutation)
76
+
77
+ @unexercised.include?(key(mutation)) ? SENTINEL_OPERATOR : nil
78
+ end
79
+
80
+ private
81
+
82
+ # Whether the subject is a method whose body raise reaches its caller: a
83
+ # method definition without a rescue or ensure clause of its own, which
84
+ # would make its body a BeginNode. Read from the subject's source, as its
85
+ # node is released before the mutations run.
86
+ def probeable?(subject)
87
+ @probeable.fetch(subject) do
88
+ definition = Prism.parse(subject.source).value.statements.body.first
89
+ @probeable[subject] = definition.is_a?(Prism::DefNode) && !definition.body.is_a?(Prism::BeginNode)
90
+ end
91
+ end
92
+
93
+ # The line tells apart two methods of one name in a file, such as a
94
+ # method defined again where its class is reopened.
95
+ def key(mutation)
96
+ subject = mutation.subject
97
+ [mutation.file_path, subject.name, subject.line_number]
98
+ end
99
+ end