evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+
5
+ # A run in two batches: first every subject's sentinel (see ProbeFirst), then
6
+ # the other mutations, once the probe knows which subjects no test exercises
7
+ # and the runless verdict can report their mutations without a run. Each
8
+ # batch is run by the block, as a strategy runs a whole run, with a notifier
9
+ # that numbers its results after the first batch's, so progress, fail-fast and
10
+ # the logs see one run. The results come back in the order the mutations came
11
+ # in.
12
+ #
13
+ # The second batch waits for the slowest sentinel: a parallel run's workers
14
+ # idle at the end of the first batch.
15
+ class Evilution::Runner::MutationExecutor::ProbeFirstRun
16
+ # The run's notifier as one batch sees it: started and finished once for the
17
+ # whole run, here, and counting on from the results reported before.
18
+ class Continuation
19
+ def initialize(notifier, offset)
20
+ @notifier = notifier
21
+ @offset = offset
22
+ end
23
+
24
+ def start(_total); end
25
+
26
+ def finish; end
27
+
28
+ def notify(result, index)
29
+ @notifier.notify(result, @offset + index)
30
+ end
31
+ end
32
+
33
+ def initialize(probe:, notifier:)
34
+ @probe = probe
35
+ @notifier = notifier
36
+ end
37
+
38
+ def call(mutations, &run_batch)
39
+ sentinels, rest = @probe.split(mutations)
40
+ @notifier.start(mutations.length)
41
+ first = run_batch.call(sentinels, Continuation.new(@notifier, 0))
42
+ first.results.each { |result| @probe.record(result) }
43
+ second = first.truncated ? nil : run_batch.call(rest, Continuation.new(@notifier, first.results.length))
44
+ @notifier.finish
45
+ merge(mutations, [first, second].compact)
46
+ end
47
+
48
+ private
49
+
50
+ def merge(mutations, executions)
51
+ position = mutations.each_with_index.to_h
52
+ Evilution::Runner::MutationExecutor::ExecutionResult.new(
53
+ results: executions.flat_map(&:results).sort_by { |result| position.fetch(result.mutation) },
54
+ truncated: executions.any?(&:truncated),
55
+ infra_retried: executions.sum(&:infra_retried),
56
+ worker_stats: executions.flat_map(&:worker_stats)
57
+ )
58
+ end
59
+ end
@@ -2,10 +2,14 @@
2
2
 
3
3
  require_relative "../mutation_executor"
4
4
  require_relative "../../result/mutation_result"
5
- require_relative "neutralizer/infra_error"
5
+ require_relative "../../infra_crash"
6
+ require_relative "runless_verdict"
6
7
 
7
8
  class Evilution::Runner::MutationExecutor::ResultCache
8
- CACHEABLE_STATUSES = %i[killed timeout].freeze
9
+ # The verdicts about a mutation. A survivor is kept too: its key changes
10
+ # with the subject and its spec files, so a test that would now kill it
11
+ # re-runs it.
12
+ CACHEABLE_STATUSES = %i[killed survived timeout].freeze
9
13
  private_constant :CACHEABLE_STATUSES
10
14
 
11
15
  Partition = Data.define(:uncached_indices, :cached_results)
@@ -14,25 +18,28 @@ class Evilution::Runner::MutationExecutor::ResultCache
14
18
  @backend = backend
15
19
  end
16
20
 
17
- def fetch(mutation)
21
+ # time_limit: the longest the mutation's test run would be allowed now; nil
22
+ # when it would run without a limit.
23
+ def fetch(mutation, time_limit:)
18
24
  return nil unless @backend
19
25
 
20
26
  data = @backend.fetch(mutation)
21
27
  return nil unless data
22
- return nil unless CACHEABLE_STATUSES.include?(data[:status])
28
+ return nil unless reusable?(data, time_limit)
23
29
 
24
30
  Evilution::Result::MutationResult.new(
25
31
  mutation: mutation,
26
32
  status: data[:status],
27
33
  duration: data[:duration],
28
34
  killing_test: data[:killing_test],
29
- test_command: data[:test_command]
35
+ test_command: data[:test_command],
36
+ time_limit: data[:time_limit]
30
37
  )
31
38
  end
32
39
 
33
40
  def store(mutation, result)
34
41
  return unless @backend
35
- return unless result.killed? || result.timeout?
42
+ return unless CACHEABLE_STATUSES.include?(result.status)
36
43
  return if infra_crash?(result)
37
44
  # Nor is one where only already-failing examples failed: the baseline
38
45
  # that says so belongs to this run, and the cache keeps no trace of it.
@@ -42,7 +49,15 @@ class Evilution::Runner::MutationExecutor::ResultCache
42
49
  status: result.status,
43
50
  duration: result.duration,
44
51
  killing_test: result.killing_test,
45
- test_command: result.test_command)
52
+ test_command: result.test_command,
53
+ time_limit: result.time_limit)
54
+ end
55
+
56
+ # Writes out what the run stored (see Evilution::Cache#flush).
57
+ def flush
58
+ return unless @backend
59
+
60
+ @backend.flush
46
61
  end
47
62
 
48
63
  # A kill that came from an infrastructure crash is not a verdict about the
@@ -51,20 +66,24 @@ class Evilution::Runner::MutationExecutor::ResultCache
51
66
  # would defeat both the demotion to neutral and the serial retry
52
67
  # (EV-j0bv / GH #1607).
53
68
  def infra_crash?(result)
54
- Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_crash_class?(result.error_class)
69
+ Evilution::InfraCrash.class_name?(result.error_class)
55
70
  end
56
71
 
57
- def partition(batch, packer:)
72
+ # time_limit_for: answers the time limit #fetch is to compare against for
73
+ # each mutation. A mutation the runless verdict answers goes in with the
74
+ # cached results: it needs no worker, and its verdict is not stored.
75
+ def partition(batch, packer:, time_limit_for:, runless: Evilution::Runner::MutationExecutor::RunlessVerdict.new)
58
76
  uncached_indices = []
59
77
  cached_results = {}
60
78
 
61
79
  batch.each_with_index do |mutation, i|
62
- if mutation.unparseable?
63
- cached_results[i] = packer.compact(unparseable_result(mutation))
80
+ verdict = runless.call(mutation)
81
+ if verdict
82
+ cached_results[i] = packer.compact(verdict)
64
83
  next
65
84
  end
66
85
 
67
- cached = fetch(mutation)
86
+ cached = fetch(mutation, time_limit: time_limit_for.call(mutation))
68
87
  if cached
69
88
  cached_results[i] = packer.compact(cached)
70
89
  else
@@ -77,7 +96,19 @@ class Evilution::Runner::MutationExecutor::ResultCache
77
96
 
78
97
  private
79
98
 
80
- def unparseable_result(mutation)
81
- Evilution::Result::MutationResult.new(mutation: mutation, status: :unparseable)
99
+ # A kill or a survivor stands whatever the limit is now. A timeout says only
100
+ # that the run needed more than the limit it had, so it answers for a run
101
+ # that would get no more time than that; one that does not record its limit
102
+ # answers for none.
103
+ def reusable?(data, time_limit)
104
+ case data[:status]
105
+ when :killed, :survived then true
106
+ when :timeout then timeout_reusable?(data[:time_limit], time_limit)
107
+ else false
108
+ end
109
+ end
110
+
111
+ def timeout_reusable?(cached_limit, time_limit)
112
+ cached_limit.is_a?(Numeric) && time_limit.is_a?(Numeric) && time_limit <= cached_limit
82
113
  end
83
114
  end
@@ -24,7 +24,7 @@ class Evilution::Runner::MutationExecutor::ResultNotifier
24
24
  @progress_bar.tick(status: result.status) if @progress_bar
25
25
  @diagnostics.log_progress(index, result.status)
26
26
  @diagnostics.log_mutation_diagnostics(result)
27
- @survived_count += 1 if result.survived?
27
+ @survived_count += 1 if result.survived? || result.uncovered?
28
28
  truncate? ? :truncate : :continue
29
29
  end
30
30
 
@@ -5,19 +5,7 @@ require_relative "../../result/mutation_result"
5
5
 
6
6
  class Evilution::Runner::MutationExecutor::ResultPacker
7
7
  def compact(result)
8
- {
9
- status: result.status,
10
- duration: result.duration,
11
- killing_test: result.killing_test,
12
- test_command: result.test_command,
13
- child_rss_kb: result.child_rss_kb,
14
- memory_delta_kb: result.memory_delta_kb,
15
- parent_rss_kb: result.parent_rss_kb,
16
- error_message: result.error_message,
17
- error_class: result.error_class,
18
- error_backtrace: result.error_backtrace,
19
- known_failures_only: result.known_failures_only?
20
- }
8
+ run_fields(result).merge(memory_fields(result), error_fields(result))
21
9
  end
22
10
 
23
11
  def rebuild(mutation, data)
@@ -27,17 +15,59 @@ class Evilution::Runner::MutationExecutor::ResultPacker
27
15
  duration: data[:duration],
28
16
  killing_test: data[:killing_test],
29
17
  test_command: data[:test_command],
30
- memory: Evilution::Result::MemoryStats.from_fields(
31
- child_rss_kb: data[:child_rss_kb],
32
- memory_delta_kb: data[:memory_delta_kb],
33
- parent_rss_kb: data[:parent_rss_kb]
34
- ),
35
- error: Evilution::Result::ErrorInfo.from_fields(
36
- message: data[:error_message],
37
- klass: data[:error_class],
38
- backtrace: data[:error_backtrace]
39
- ),
40
- known_failures_only: data[:known_failures_only]
18
+ memory: rebuild_memory(data),
19
+ error: rebuild_error(data),
20
+ known_failures_only: data[:known_failures_only],
21
+ timings: rebuild_timings(data),
22
+ time_limit: data[:time_limit],
23
+ examples_run: data[:examples_run],
24
+ implied_by: data[:implied_by]
25
+ )
26
+ end
27
+
28
+ private
29
+
30
+ def run_fields(result)
31
+ {
32
+ status: result.status,
33
+ duration: result.duration,
34
+ killing_test: result.killing_test,
35
+ test_command: result.test_command,
36
+ known_failures_only: result.known_failures_only?,
37
+ timings: result.timings ? result.timings.to_h : nil,
38
+ time_limit: result.time_limit,
39
+ examples_run: result.examples_run,
40
+ implied_by: result.implied_by
41
+ }
42
+ end
43
+
44
+ def memory_fields(result)
45
+ { child_rss_kb: result.child_rss_kb, memory_delta_kb: result.memory_delta_kb, parent_rss_kb: result.parent_rss_kb }
46
+ end
47
+
48
+ def error_fields(result)
49
+ { error_message: result.error_message, error_class: result.error_class, error_backtrace: result.error_backtrace }
50
+ end
51
+
52
+ def rebuild_memory(data)
53
+ Evilution::Result::MemoryStats.from_fields(
54
+ child_rss_kb: data[:child_rss_kb], memory_delta_kb: data[:memory_delta_kb], parent_rss_kb: data[:parent_rss_kb]
55
+ )
56
+ end
57
+
58
+ def rebuild_error(data)
59
+ Evilution::Result::ErrorInfo.from_fields(
60
+ message: data[:error_message], klass: data[:error_class], backtrace: data[:error_backtrace]
41
61
  )
42
62
  end
63
+
64
+ # queue_wait is measured in the worker around the isolated run, so it
65
+ # arrives beside the result's own timings rather than inside them.
66
+ def rebuild_timings(data)
67
+ timings = Evilution::Result::MutationTimings.from_h(data[:timings])
68
+ wait = data[:queue_wait]
69
+ return timings unless wait.is_a?(Numeric)
70
+
71
+ timings ? timings.with_queue_wait(wait) : Evilution::Result::MutationTimings.new(queue_wait: wait)
72
+ end
43
73
  end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+ require_relative "../../result/mutation_result"
5
+ require_relative "probe_first"
6
+
7
+ # The verdict a mutation gets without a test run, when one can be given:
8
+ # its mutated source does not parse, no test executes any line it changes
9
+ # (asked of `coverage`, an integration backed by a coverage map), or its
10
+ # subject's sentinel survived (asked of `probe`, see ProbeFirst), which
11
+ # implies it survives too. nil for every other mutation, which runs.
12
+ #
13
+ # A mutation no test executes could only survive, and a survivor costs the
14
+ # most of any verdict: the whole resolved spec file, then a confirmation run.
15
+ class Evilution::Runner::MutationExecutor::RunlessVerdict
16
+ def initialize(coverage: nil, probe: Evilution::Runner::MutationExecutor::ProbeFirst::NONE)
17
+ @coverage = coverage
18
+ @probe = probe
19
+ end
20
+
21
+ def call(mutation)
22
+ return result(mutation, :unparseable) if mutation.unparseable?
23
+ return result(mutation, :uncovered) if !@coverage.nil? && @coverage.uncovered?(mutation)
24
+
25
+ implied(mutation, @probe.implied_by(mutation))
26
+ end
27
+
28
+ private
29
+
30
+ def result(mutation, status)
31
+ Evilution::Result::MutationResult.new(mutation: mutation, status: status)
32
+ end
33
+
34
+ def implied(mutation, sentinel_operator)
35
+ return nil if sentinel_operator.nil?
36
+
37
+ Evilution::Result::MutationResult.new(mutation: mutation, status: :survived, implied_by: sentinel_operator)
38
+ end
39
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../parallel"
4
+
5
+ # The mutations of a parallel run as the pool sends them out: each paired
6
+ # with the tests that killed its neighbours (see KillHistory), looked up when
7
+ # the pool takes it, not when the run began, so it carries every kill reported
8
+ # by then.
9
+ class Evilution::Runner::MutationExecutor::Strategy::Parallel::HintedItems
10
+ def initialize(mutations, kill_history)
11
+ @mutations = mutations
12
+ @kill_history = kill_history
13
+ end
14
+
15
+ def length
16
+ @mutations.length
17
+ end
18
+
19
+ def empty?
20
+ @mutations.empty?
21
+ end
22
+
23
+ def [](index)
24
+ mutation = @mutations[index]
25
+ [mutation, @kill_history.killers_for(mutation)]
26
+ end
27
+ end
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../parallel"
4
+ require_relative "../../../../infra_crash"
5
+
6
+ # Turns the compact results of a parallel run, which arrive in whatever order
7
+ # the workers finish, into the run's results in mutation order. A result is
8
+ # held back until every earlier mutation has one, then rebuilt, cached when the
9
+ # workers produced it in this run, has its mutation's sources dropped, goes
10
+ # through the neutralisation pipeline and on to the notifier, so what the
11
+ # notifier sees, where fail-fast cuts the run and what the reports list are the
12
+ # same as one mutation after another would give.
13
+ class Evilution::Runner::MutationExecutor::Strategy::Parallel::OrderedDelivery
14
+ MEMORY_LOG_LABEL = "after batch"
15
+
16
+ attr_reader :results
17
+
18
+ # `log_every` is how many results go by between two memory logs.
19
+ def initialize(mutations:, packer:, cache:, pipeline:, notifier:, baseline_result:, diagnostics:, log_every:)
20
+ @mutations = mutations
21
+ @packer = packer
22
+ @cache = cache
23
+ @pipeline = pipeline
24
+ @notifier = notifier
25
+ @baseline_result = baseline_result
26
+ @diagnostics = diagnostics
27
+ @log_every = log_every
28
+ @pending = {}
29
+ @results = []
30
+ @truncated = false
31
+ end
32
+
33
+ def truncated?
34
+ @truncated
35
+ end
36
+
37
+ # `fresh` says the workers produced the result in this run, as opposed to it
38
+ # coming from the cache.
39
+ # Results held back or arriving once the run is truncated are neither listed
40
+ # nor notified. The fresh ones still go to the cache: they are verdicts the
41
+ # workers spent the time on, and a later --incremental run need not repeat
42
+ # them.
43
+ def accept(index, compact, fresh:)
44
+ @pending[index] = [compact, fresh]
45
+ release while !@truncated && @pending.key?(@results.length)
46
+ cache_held_back if @truncated
47
+ end
48
+
49
+ private
50
+
51
+ def cache_held_back
52
+ @pending.each do |index, (compact, fresh)|
53
+ next unless fresh
54
+
55
+ mutation = @mutations[index]
56
+ @cache.store(mutation, @packer.rebuild(mutation, compact))
57
+ end
58
+ @pending.clear
59
+ end
60
+
61
+ def release
62
+ mutation = @mutations[@results.length]
63
+ compact, fresh = @pending.delete(@results.length)
64
+ result = @packer.rebuild(mutation, compact)
65
+ @cache.store(mutation, result) if fresh
66
+ release_sources(mutation, result)
67
+ record(@pipeline.call(result, baseline_result: @baseline_result))
68
+ end
69
+
70
+ # Sources are dropped as soon as a mutation has been judged, to keep a long
71
+ # run's memory flat. A mutation the workers could not judge because of
72
+ # infrastructure contention is re-run serially once the pool is done
73
+ # (InfraRetry), which needs its sources, so it keeps them until then. The
74
+ # neutralisation pipeline has not run yet here, so the crash is still a kill.
75
+ def release_sources(mutation, result)
76
+ return if Evilution::InfraCrash.class_name?(result.error_class)
77
+
78
+ mutation.strip_sources!
79
+ end
80
+
81
+ def record(result)
82
+ @results << result
83
+ @truncated = @notifier.notify(result, @results.length) == :truncate
84
+ log_memory
85
+ end
86
+
87
+ def log_memory
88
+ return unless @diagnostics
89
+ return unless log_due?
90
+
91
+ @diagnostics.log_memory(MEMORY_LOG_LABEL, "#{@results.length} complete")
92
+ end
93
+
94
+ def log_due?
95
+ @truncated || (@results.length % @log_every).zero? || @results.length == @mutations.length
96
+ end
97
+ end