evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -2,8 +2,19 @@
2
2
 
3
3
  require_relative "../strategy"
4
4
  require_relative "../../../parallel/work_queue"
5
+ require_relative "../timeout_policy"
6
+ require_relative "../runless_verdict"
7
+ require_relative "../kill_history"
8
+ require_relative "../timed_run"
5
9
 
6
10
  class Evilution::Runner::MutationExecutor::Strategy::Parallel
11
+ # Autoloaded rather than required at the bottom: ordered_delivery requires
12
+ # this file, and requiring it while it is still loading is a circular
13
+ # require, which Ruby warns about -- and a project that turns warnings into
14
+ # exceptions loads this after its spec helper did so.
15
+ autoload :OrderedDelivery, File.expand_path("parallel/ordered_delivery", __dir__)
16
+ autoload :HintedItems, File.expand_path("parallel/hinted_items", __dir__)
17
+
7
18
  # Compact result for a worker that exited unexpectedly (:error is not cached,
8
19
  # so the 0.0 duration is never persisted).
9
20
  DIED_COMPACT = {
@@ -22,7 +33,16 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
22
33
  error_class: "Evilution::Error"
23
34
  }.freeze
24
35
 
25
- def initialize(cache:, isolator:, packer:, pipeline:, notifier:, pool_factory:, config:, diagnostics: nil)
36
+ # pool_factory: called with the pool's item timeout, the backstop for a
37
+ # worker that stops answering. timeout_policy: the limit each mutation runs
38
+ # under; the configured timeout for every one of them when not given.
39
+ # runless: the verdict a mutation gets without a run, when it has one.
40
+ # kill_history: the tests that killed earlier mutations of the run (see
41
+ # KillHistory); each mutation goes to its worker with those of its
42
+ # neighbours, and each kill that comes back is added.
43
+ def initialize(cache:, isolator:, packer:, pipeline:, notifier:, pool_factory:, config:, diagnostics: nil,
44
+ timeout_policy: nil, runless: Evilution::Runner::MutationExecutor::RunlessVerdict.new,
45
+ kill_history: Evilution::Runner::MutationExecutor::KillHistory::NONE)
26
46
  @cache = cache
27
47
  @isolator = isolator
28
48
  @packer = packer
@@ -31,125 +51,124 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
31
51
  @pool_factory = pool_factory
32
52
  @diagnostics = diagnostics
33
53
  @config = config
54
+ @timeout_policy = timeout_policy || Evilution::Runner::MutationExecutor::TimeoutPolicy::Fixed.new(config.timeout)
55
+ @runless = runless
56
+ @kill_history = kill_history
57
+ @timed_run = Evilution::Runner::MutationExecutor::TimedRun.new(isolator: isolator, timeout_policy: @timeout_policy)
34
58
  end
35
59
 
36
60
  def call(mutations, baseline_result:, integration:)
37
61
  @notifier.start(mutations.length)
38
- pool = @pool_factory.call
39
- state = { results: [], truncated: false, completed: 0 }
40
- all_worker_stats = run_batches(mutations, pool, baseline_result, integration, state)
62
+ delivery = build_delivery(mutations, baseline_result)
63
+ worker_stats = Evilution::Parallel::WorkQueue::WorkerStat.aggregate(run(mutations, delivery, integration))
41
64
 
42
- log_worker_diagnostics(all_worker_stats)
65
+ log_worker_diagnostics(worker_stats)
43
66
  @notifier.finish
44
- build_result(state)
67
+ Evilution::Runner::MutationExecutor::ExecutionResult.new(
68
+ results: delivery.results, truncated: delivery.truncated?, worker_stats: worker_stats
69
+ )
45
70
  end
46
71
 
47
72
  private
48
73
 
49
- def run_batches(mutations, pool, baseline_result, integration, state)
50
- all_worker_stats = []
51
- mutations.each_slice(@config.jobs) do |batch|
52
- break if state[:truncated]
53
-
54
- batch_results = run_batch(batch, pool, integration)
55
- all_worker_stats.concat(pool.worker_stats)
56
- process_batch(batch_results, baseline_result, state)
57
- end
58
- all_worker_stats
74
+ def build_delivery(mutations, baseline_result)
75
+ OrderedDelivery.new(
76
+ mutations: mutations, packer: @packer, cache: @cache, pipeline: @pipeline, notifier: @notifier,
77
+ baseline_result: baseline_result, diagnostics: @diagnostics, log_every: @config.jobs
78
+ )
59
79
  end
60
80
 
61
- def log_worker_diagnostics(all_worker_stats)
62
- return unless @diagnostics
63
-
64
- @diagnostics.log_worker_stats(@diagnostics.aggregate_worker_stats(all_worker_stats))
65
- end
81
+ # The cached results, and the verdicts given without a run, go in first;
82
+ # the rest run on one pool whose workers
83
+ # live for the whole run and take the next mutation as soon as they are done
84
+ # with the last, so a slow mutation holds up only its own worker.
85
+ def run(mutations, delivery, integration)
86
+ partition = @cache.partition(mutations, packer: @packer, runless: @runless,
87
+ time_limit_for: @timeout_policy.method(:final_limit))
88
+ partition.cached_results.each do |index, compact|
89
+ remember(mutations[index], compact)
90
+ delivery.accept(index, compact, fresh: false)
91
+ end
92
+ return [] if delivery.truncated? || partition.uncached_indices.empty?
66
93
 
67
- def build_result(state)
68
- Evilution::Runner::MutationExecutor::ExecutionResult.new(results: state[:results], truncated: state[:truncated])
94
+ run_uncached(mutations, partition.uncached_indices, delivery, integration)
69
95
  end
70
96
 
71
- def run_batch(batch, pool, integration)
72
- partition = @cache.partition(batch, packer: @packer)
73
- worker_results = run_uncached(batch, partition.uncached_indices, pool, integration)
74
- compact_results = merge(batch, partition.uncached_indices, partition.cached_results, worker_results)
75
- batch_results = rebuild_results(batch, compact_results)
76
- cache_results(batch_results, partition.uncached_indices)
77
- release_sources(batch, batch_results)
78
- batch_results
97
+ # Returns the pool's worker figures once every worker has been wound down.
98
+ def run_uncached(mutations, uncached_indices, delivery, integration)
99
+ uncached = uncached_indices.map { |i| mutations[i] }
100
+ item_timeout = pool_deadline(uncached)
101
+ pool = @pool_factory.call(item_timeout)
102
+ on_result = lambda do |position, raw|
103
+ delivery.accept(uncached_indices[position], unpack_unfinished(raw, item_timeout, uncached[position]), fresh: true)
104
+ :stop if delivery.truncated?
105
+ end
106
+ run_pool(pool, uncached, on_result, arrival(uncached, item_timeout), integration)
79
107
  end
80
108
 
81
- # Sources are dropped as soon as a mutation has been judged, to keep a long
82
- # run's memory flat. A mutation the workers could not judge because of
83
- # infrastructure contention is re-run serially once the pool is done
84
- # (InfraRetry), which needs its sources, so it keeps them until then. The
85
- # neutralisation pipeline has not run yet here, so the crash is still a kill.
86
- def release_sources(batch, batch_results)
87
- batch.each_with_index do |mutation, index|
88
- result = batch_results[index]
89
- next if result && infra_crash?(result)
90
-
91
- mutation.strip_sources!
109
+ def run_pool(pool, uncached, on_result, on_arrival, integration)
110
+ items = HintedItems.new(uncached, @kill_history)
111
+ pool.map(items, on_result: on_result, on_arrival: on_arrival) do |(mutation, killers), waited|
112
+ run_in_worker(mutation, killers, integration, waited)
92
113
  end
114
+ pool.worker_stats
93
115
  end
94
116
 
95
- def infra_crash?(result)
96
- Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
117
+ # A kill is remembered the moment it is read, before its worker is sent the
118
+ # next mutation, which then goes out with it.
119
+ def arrival(uncached, item_timeout)
120
+ ->(position, raw) { remember(uncached[position], unpack_unfinished(raw, item_timeout, uncached[position])) }
97
121
  end
98
122
 
99
- def rebuild_results(batch, compact_results)
100
- batch.zip(compact_results).map { |m, h| @packer.rebuild(m, h) }
123
+ def remember(mutation, compact)
124
+ @kill_history.record(mutation, status: compact[:status], killing_test: compact[:killing_test])
101
125
  end
102
126
 
103
- def cache_results(batch_results, uncached_indices)
104
- uncached_indices.each { |i| @cache.store(batch_results[i].mutation, batch_results[i]) }
127
+ # How long the pool waits on a worker for one mutation. The policy's item
128
+ # timeout covers a run under the adaptive limit and its confirmation under
129
+ # the fixed one (see TimedRun). A run that timed out in the killer-first
130
+ # order runs again in the defined order under the same limit (see
131
+ # IsolatedRun), in the same worker, so the wait covers both of each.
132
+ def pool_deadline(uncached)
133
+ deadline = @timeout_policy.item_timeout(uncached)
134
+ return deadline unless deadline && @config.killer_first?
135
+
136
+ deadline * 2
105
137
  end
106
138
 
107
- def run_uncached(batch, uncached_indices, pool, integration)
108
- return [] if uncached_indices.empty?
139
+ def log_worker_diagnostics(worker_stats)
140
+ return unless @diagnostics
109
141
 
110
- uncached = uncached_indices.map { |i| batch[i] }
111
- worker_results = pool.map(uncached) do |mutation|
112
- test_command = ->(m) { integration.call(m) }
113
- result = @isolator.call(mutation: mutation, test_command: test_command, timeout: @config.timeout)
114
- @packer.compact(result)
115
- end
116
- worker_results.map { |r| unpack_unfinished(r) }
142
+ @diagnostics.log_worker_stats(worker_stats)
143
+ end
144
+
145
+ # Runs in the worker. `waited` is how long the mutation sat between the
146
+ # dispatcher sending it and this worker starting it, nil when not measured.
147
+ # The integration is told the killers before the isolator runs, so a forked
148
+ # child inherits what it was told.
149
+ def run_in_worker(mutation, killers, integration, waited)
150
+ integration.prioritize(killers) if integration.respond_to?(:prioritize)
151
+ result = @timed_run.call(mutation, integration)
152
+ @packer.compact(result).merge(queue_wait: waited)
117
153
  end
118
154
 
119
- def unpack_unfinished(result)
155
+ def unpack_unfinished(result, item_timeout, mutation)
120
156
  return result unless result.is_a?(Evilution::Parallel::WorkQueue::Unfinished)
121
157
 
122
158
  case result.reason
123
- when :timeout then timeout_compact
159
+ when :timeout then timeout_compact(item_timeout, @timeout_policy.final_limit(mutation))
124
160
  when :died then DIED_COMPACT
125
161
  when :unreadable then UNREADABLE_COMPACT
126
162
  end
127
163
  end
128
164
 
129
165
  # A :timeout result is cacheable, so give it a realistic duration: the stuck
130
- # worker exhausted the dispatcher's item_timeout (config.timeout * 2) before
131
- # being killed. A 0.0 here would be cached and skew summaries/reuse.
132
- def timeout_compact
133
- { status: :timeout, duration: @config.timeout * 2.0 }
134
- end
135
-
136
- def merge(batch, uncached_indices, cached_results, worker_results)
137
- result_map = cached_results.dup
138
- uncached_indices.each_with_index { |batch_idx, worker_idx| result_map[batch_idx] = worker_results[worker_idx] }
139
- batch.each_index.map { |i| result_map[i] }
140
- end
141
-
142
- def process_batch(batch_results, baseline_result, state)
143
- batch_results.each do |result|
144
- result = @pipeline.call(result, baseline_result: baseline_result)
145
- state[:results] << result
146
- state[:completed] += 1
147
- if @notifier.notify(result, state[:completed]) == :truncate
148
- state[:truncated] = true
149
- break
150
- end
151
- end
152
-
153
- @diagnostics.log_memory("after batch", "#{state[:completed]} complete") if @diagnostics
166
+ # worker exhausted the dispatcher's item_timeout before being killed. A 0.0
167
+ # here would be cached and skew summaries/reuse. Only a pool that has an
168
+ # item timeout times an item out, so it is never nil here. The limit is the
169
+ # mutation's own, which the worker should have enforced, not the backstop
170
+ # the pool set from the longest mutation of the run.
171
+ def timeout_compact(item_timeout, time_limit)
172
+ { status: :timeout, duration: item_timeout, timings: { test_run: item_timeout }, time_limit: time_limit }
154
173
  end
155
174
  end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+ require_relative "isolated_run"
5
+
6
+ # Runs a mutation's tests under the limit its timeout policy gives it. A
7
+ # timeout the policy wants confirmed is not reported as it is: the mutation is
8
+ # run again under the confirmation limit, and that run's verdict stands, with
9
+ # the time of the first run added to what the mutation cost.
10
+ #
11
+ # Each run under a limit is an IsolatedRun, so a run whose tests the
12
+ # integration reordered and that timed out is run again in the defined order
13
+ # under the same limit before the confirmation is asked for: a mutation that
14
+ # times out every way is judged by a run in the defined order under the
15
+ # confirmation limit.
16
+ class Evilution::Runner::MutationExecutor::TimedRun
17
+ def initialize(isolator:, timeout_policy:)
18
+ @isolated_run = Evilution::Runner::MutationExecutor::IsolatedRun.new(isolator)
19
+ @timeout_policy = timeout_policy
20
+ end
21
+
22
+ # integration: what runs the tests, answering #call(mutation) with the test
23
+ # command (see IsolatedRun).
24
+ def call(mutation, integration)
25
+ result = @isolated_run.call(mutation, integration, @timeout_policy.call(mutation))
26
+ return result unless result.timeout?
27
+
28
+ confirmation = @timeout_policy.confirmation_limit(mutation)
29
+ return result unless confirmation
30
+
31
+ @isolated_run.call(mutation, integration, confirmation).after_adaptive_timeout(result.duration)
32
+ end
33
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ # A limit per mutation of `factor` times the baseline time of the spec files it
6
+ # runs, kept between `floor` and `ceiling`.
7
+ #
8
+ # The baseline ran each spec file in a forked process of its own, so its time
9
+ # holds the fork and the loading of the file, but none of the hooks a mutation
10
+ # run fires (the policy is not used when those are configured). The factor
11
+ # leaves room for the re-run of a survivor against its whole spec files and
12
+ # for workers competing for the CPU under --jobs. The floor covers spec files
13
+ # so fast that scheduling noise outweighs them.
14
+ #
15
+ # None of that bounds a mutant that slows the code down and still fails its
16
+ # test: such a kill can come after any limit derived from the baseline. A
17
+ # timeout under a limit shorter than the fallback is therefore confirmed by
18
+ # running the mutation again under the fallback limit, so a timeout is only
19
+ # reported where the fallback limit would have reported one too.
20
+ #
21
+ # A mutation keeps the fallback limit when the baseline cannot vouch for its
22
+ # spec files: none resolve (the run would fall back to the whole suite, or
23
+ # report it unresolved), one of them has no baseline time, or one of them
24
+ # failed in the baseline and so may have stopped early.
25
+ class Evilution::Runner::MutationExecutor::TimeoutPolicy::Adaptive < Evilution::Runner::MutationExecutor::TimeoutPolicy::Base
26
+ def initialize(fallback:, spec_durations:, failed_spec_files:, spec_selector:, factor:, floor:, ceiling:)
27
+ super()
28
+ @fallback = fallback
29
+ @spec_durations = spec_durations
30
+ @failed_spec_files = failed_spec_files
31
+ @spec_selector = spec_selector
32
+ @factor = factor
33
+ @floor = floor
34
+ @ceiling = ceiling
35
+ end
36
+
37
+ def call(mutation)
38
+ adaptive_limit(mutation) || @fallback.call(mutation)
39
+ end
40
+
41
+ def confirmation_limit(mutation)
42
+ limit = adaptive_limit(mutation)
43
+ return nil if limit.nil?
44
+
45
+ fallback = @fallback.call(mutation)
46
+ fallback if fallback && limit < fallback
47
+ end
48
+
49
+ private
50
+
51
+ def adaptive_limit(mutation)
52
+ seconds = baseline_seconds(Array(@spec_selector.call(mutation.file_path)))
53
+ return nil if seconds.nil?
54
+
55
+ (seconds * @factor).clamp(@floor, @ceiling)
56
+ end
57
+
58
+ def baseline_seconds(spec_files)
59
+ return nil if spec_files.empty?
60
+ return nil if spec_files.any? { |spec_file| @failed_spec_files.include?(spec_file) }
61
+
62
+ durations = spec_files.map { |spec_file| @spec_durations[spec_file] }
63
+ return nil if durations.any?(nil)
64
+
65
+ durations.sum
66
+ end
67
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../timeout_policy"
4
+
5
+ class Evilution::Runner::MutationExecutor::TimeoutPolicy::Base
6
+ # The limit a timeout under #call is confirmed under before it is reported,
7
+ # by running the mutation again; nil when a timeout under #call stands.
8
+ def confirmation_limit(_mutation)
9
+ nil
10
+ end
11
+
12
+ # The limit a timeout this mutation reports has used up: the longest its
13
+ # test run is allowed.
14
+ def final_limit(mutation)
15
+ confirmation_limit(mutation) || call(mutation)
16
+ end
17
+
18
+ # Seconds the pool waits on any one of these mutations before it gives the
19
+ # worker up: a multiple of the longest one of them may take, a timed-out run
20
+ # and its confirmation together; nil when they run without a limit.
21
+ def item_timeout(mutations)
22
+ longest = mutations.map { |mutation| longest_run(mutation) }
23
+ return nil if longest.empty? || longest.include?(nil)
24
+
25
+ longest.max * Evilution::Runner::MutationExecutor::TimeoutPolicy::ITEM_TIMEOUT_FACTOR
26
+ end
27
+
28
+ private
29
+
30
+ def longest_run(mutation)
31
+ limit = call(mutation)
32
+ confirmation = confirmation_limit(mutation)
33
+ confirmation ? limit + confirmation : limit
34
+ end
35
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ # The same limit for every mutation: the timeout the user gave, or the default
6
+ # when nothing better is known. nil runs without a limit.
7
+ class Evilution::Runner::MutationExecutor::TimeoutPolicy::Fixed < Evilution::Runner::MutationExecutor::TimeoutPolicy::Base
8
+ def initialize(seconds)
9
+ super()
10
+ @seconds = seconds
11
+ freeze
12
+ end
13
+
14
+ def call(_mutation)
15
+ @seconds
16
+ end
17
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+
5
+ # Chooses the time limit each mutation's test run gets.
6
+ #
7
+ # One fixed limit makes a mutant that loops cost the whole limit even when its
8
+ # spec file passes in a fraction of a second. When the run measured a baseline,
9
+ # the limit follows the time the mutation's own spec files took there instead
10
+ # (Adaptive). A timeout the user gave keeps winning, and anything the baseline
11
+ # cannot vouch for keeps the fixed limit.
12
+ module Evilution::Runner::MutationExecutor::TimeoutPolicy
13
+ # The pool's deadline for an item is a backstop for a worker that stops
14
+ # answering, so it sits well past the limit the worker enforces itself.
15
+ ITEM_TIMEOUT_FACTOR = 2
16
+
17
+ # Hooks that run inside the time a mutation's test run is allowed. The
18
+ # baseline fires none of them, so its times leave their cost out.
19
+ PER_MUTATION_HOOKS = %w[worker_process_start mutation_insert_pre mutation_insert_post].freeze
20
+
21
+ # Autoloaded rather than required at the bottom: each policy file requires
22
+ # this one, and requiring it while it is still loading is a circular require,
23
+ # which Ruby warns about -- and a project that turns warnings into exceptions
24
+ # loads this after its spec helper did so.
25
+ autoload :Base, File.expand_path("timeout_policy/base", __dir__)
26
+ autoload :Fixed, File.expand_path("timeout_policy/fixed", __dir__)
27
+ autoload :Adaptive, File.expand_path("timeout_policy/adaptive", __dir__)
28
+
29
+ def self.build(config, baseline_result)
30
+ fixed = Fixed.new(config.timeout)
31
+ return fixed unless adaptive?(config, baseline_result)
32
+
33
+ Adaptive.new(
34
+ fallback: fixed,
35
+ spec_durations: baseline_result.spec_file_durations,
36
+ failed_spec_files: baseline_result.failed_spec_files,
37
+ spec_selector: config.spec_selector,
38
+ factor: config.timeout_factor,
39
+ floor: config.timeout_floor,
40
+ ceiling: config.timeout_ceiling
41
+ )
42
+ end
43
+
44
+ # The related-specs heuristic adds spec files per mutation that the baseline
45
+ # never ran, so there is no time to scale for them.
46
+ def self.adaptive?(config, baseline_result)
47
+ config.adaptive_timeout? && !baseline_result.nil? && !config.related_specs_heuristic? &&
48
+ !per_mutation_hooks?(config)
49
+ end
50
+ private_class_method :adaptive?
51
+
52
+ def self.per_mutation_hooks?(config)
53
+ config.hooks.keys.any? { |event| PER_MUTATION_HOOKS.include?(event.to_s) }
54
+ end
55
+ private_class_method :per_mutation_hooks?
56
+ end
@@ -1,10 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "../runner"
4
+ require_relative "../isolation/in_process"
4
5
 
5
6
  class Evilution::Runner::MutationExecutor
6
- ExecutionResult = Data.define(:results, :truncated, :infra_retried) do
7
- def initialize(results:, truncated:, infra_retried: 0)
7
+ ExecutionResult = Data.define(:results, :truncated, :infra_retried, :worker_stats) do
8
+ def initialize(results:, truncated:, infra_retried: 0, worker_stats: [])
8
9
  super
9
10
  end
10
11
  end
@@ -17,6 +18,13 @@ class Evilution::Runner::MutationExecutor
17
18
  autoload :InfraRetry, File.expand_path("mutation_executor/infra_retry", __dir__)
18
19
  autoload :Strategy, File.expand_path("mutation_executor/strategy", __dir__)
19
20
  autoload :Neutralizer, File.expand_path("mutation_executor/neutralizer", __dir__)
21
+ autoload :TimeoutPolicy, File.expand_path("mutation_executor/timeout_policy", __dir__)
22
+ autoload :RunlessVerdict, File.expand_path("mutation_executor/runless_verdict", __dir__)
23
+ autoload :KillHistory, File.expand_path("mutation_executor/kill_history", __dir__)
24
+ autoload :ProbeFirst, File.expand_path("mutation_executor/probe_first", __dir__)
25
+ autoload :ProbeFirstRun, File.expand_path("mutation_executor/probe_first_run", __dir__)
26
+ autoload :IsolatedRun, File.expand_path("mutation_executor/isolated_run", __dir__)
27
+ autoload :TimedRun, File.expand_path("mutation_executor/timed_run", __dir__)
20
28
 
21
29
  def initialize(config, isolator:, baseline_runner:, cache:, hooks:, diagnostics:, on_result: nil)
22
30
  @config = config
@@ -31,16 +39,49 @@ class Evilution::Runner::MutationExecutor
31
39
 
32
40
  def call(mutations, baseline_result = nil)
33
41
  integration = @baseline_runner.build_integration(baseline_result)
42
+ warm(integration, mutations)
34
43
  spec_resolver = baseline_failed?(baseline_result) ? @baseline_runner.neutralization_resolver : nil
35
44
  notifier = build_notifier
36
45
  pipeline = build_pipeline(spec_resolver)
37
- return sequential_run(notifier, pipeline, mutations, baseline_result, integration) if @config.jobs <= 1
46
+ prepare_run(baseline_result, integration)
47
+ return run_batch(notifier, pipeline, mutations, baseline_result, integration) unless @config.probe_first?
38
48
 
39
- parallel_run(notifier, pipeline, mutations, baseline_result, integration)
49
+ ProbeFirstRun.new(probe: @probe, notifier: notifier).call(mutations) do |batch, batch_notifier|
50
+ run_batch(batch_notifier, pipeline, batch, baseline_result, integration)
51
+ end
52
+ ensure
53
+ # The cache holds the verdicts stored since it last wrote a file; a run
54
+ # that stops early keeps the ones it reached.
55
+ @cache.flush
40
56
  end
41
57
 
42
58
  private
43
59
 
60
+ # What the run decides once, before any mutation: each one's time limit,
61
+ # the verdicts given without a run, and the kills remembered along the way.
62
+ def prepare_run(baseline_result, integration)
63
+ @timeout_policy = TimeoutPolicy.build(@config, baseline_result)
64
+ @probe = @config.probe_first? ? ProbeFirst.new : ProbeFirst::NONE
65
+ @runless = RunlessVerdict.new(coverage: uncovered_lines(integration), probe: @probe)
66
+ @kill_history = KillHistory.new
67
+ end
68
+
69
+ # Here, before the first mutation child or pool worker is forked, so every
70
+ # one of them inherits the filled caches rather than parsing the same source
71
+ # and spec files, or walking the same heap, for itself and losing the
72
+ # result when it exits. An integration is any callable; one without such
73
+ # caches has no hook.
74
+ def warm(integration, mutations)
75
+ integration.warm_targeting(mutations) if integration.respond_to?(:warm_targeting)
76
+ integration.warm_loading if integration.respond_to?(:warm_loading)
77
+ end
78
+
79
+ def run_batch(notifier, pipeline, mutations, baseline_result, integration)
80
+ return sequential_run(notifier, pipeline, mutations, baseline_result, integration) if @config.jobs <= 1
81
+
82
+ parallel_run(notifier, pipeline, mutations, baseline_result, integration)
83
+ end
84
+
44
85
  def sequential_run(notifier, pipeline, mutations, baseline_result, integration)
45
86
  build_sequential(notifier, pipeline)
46
87
  .call(mutations, baseline_result: baseline_result, integration: integration)
@@ -57,11 +98,20 @@ class Evilution::Runner::MutationExecutor
57
98
  results = retry_pass.call(execution.results, baseline_result: baseline_result, integration: integration)
58
99
 
59
100
  ExecutionResult.new(results: results, truncated: execution.truncated,
60
- infra_retried: retry_pass.retried_count)
101
+ infra_retried: retry_pass.retried_count, worker_stats: execution.worker_stats)
102
+ end
103
+
104
+ # What tells the mutations no test executes, when the run skips them: the
105
+ # integration, which only knows with a coverage map behind it.
106
+ def uncovered_lines(integration)
107
+ return nil unless @config.skip_uncovered? && integration.respond_to?(:uncovered?)
108
+
109
+ integration
61
110
  end
62
111
 
63
112
  def build_mutation_runner
64
- MutationRunner.new(config: @config, cache: @cache, isolator: @isolator)
113
+ MutationRunner.new(cache: @cache, isolator: @isolator, timeout_policy: @timeout_policy, runless: @runless,
114
+ kill_history: @kill_history)
65
115
  end
66
116
 
67
117
  def baseline_failed?(baseline_result)
@@ -100,17 +150,32 @@ class Evilution::Runner::MutationExecutor
100
150
  packer: @packer,
101
151
  pipeline: pipeline,
102
152
  notifier: notifier,
103
- pool_factory: -> { build_pool },
153
+ pool_factory: ->(item_timeout) { build_pool(item_timeout) },
104
154
  diagnostics: @diagnostics,
105
- config: @config
155
+ config: @config,
156
+ timeout_policy: @timeout_policy,
157
+ runless: @runless,
158
+ kill_history: @kill_history
106
159
  )
107
160
  end
108
161
 
109
- def build_pool
162
+ def build_pool(item_timeout)
110
163
  Evilution::Parallel::Pool.new(
111
164
  size: @config.jobs,
112
165
  hooks: @hooks,
113
- item_timeout: @config.timeout ? @config.timeout * 2 : nil
166
+ item_timeout: item_timeout,
167
+ worker_max_items: worker_max_items,
168
+ copy_worker_databases: @config.copy_worker_databases?
114
169
  )
115
170
  end
171
+
172
+ # Fork isolation runs each mutation in a child of the worker, so the worker
173
+ # itself stays clean and can serve the whole run. In-process isolation runs
174
+ # the mutation in the worker: whatever it leaves behind (global state,
175
+ # redefined constants, memory) would carry over to the next mutation that
176
+ # worker takes, and which mutations share a worker depends on timing. Each
177
+ # one therefore gets a fresh worker forked from the clean parent.
178
+ def worker_max_items
179
+ @isolator.is_a?(Evilution::Isolation::InProcess) ? 1 : nil
180
+ end
116
181
  end