evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -3,13 +3,14 @@
3
3
  require_relative "../runner"
4
4
  require_relative "../disable_comment"
5
5
  require_relative "../ast/sorbet_sig_detector"
6
- require_relative "../ast/pattern/filter"
6
+ require_relative "../ast/skip_filter"
7
7
  require_relative "../equivalent/detector"
8
8
 
9
9
  class Evilution::Runner::MutationPlanner
10
- Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, :mutated_lines, keyword_init: true)
10
+ Plan = Struct.new(:enabled, :equivalent, :skipped_count, :skipped_sites, :disabled_mutations, :mutated_lines,
11
+ keyword_init: true)
11
12
 
12
- GenerationResult = Data.define(:mutations, :skipped)
13
+ GenerationResult = Data.define(:mutations, :skipped, :skipped_sites)
13
14
  DisabledFilterResult = Data.define(:enabled, :disabled)
14
15
  SigFilterResult = Data.define(:enabled, :skipped)
15
16
  EquivalentFilterResult = Data.define(:equivalent, :enabled)
@@ -33,7 +34,8 @@ class Evilution::Runner::MutationPlanner
33
34
  sig_filter = filter_sig_blocks(disabled_filter.enabled)
34
35
  equivalent_filter = filter_equivalent(sig_filter.enabled)
35
36
 
36
- plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
37
+ plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter),
38
+ generation.skipped_sites)
37
39
  plan.mutated_lines = mutated_lines(deduped)
38
40
  plan
39
41
  end
@@ -42,11 +44,14 @@ class Evilution::Runner::MutationPlanner
42
44
 
43
45
  attr_reader :config, :registry
44
46
 
45
- # The lines mutations were generated for, per file. Taken before any filter:
46
- # a line whose mutations were all disabled or judged equivalent was still
47
- # reached by an operator.
47
+ # The lines mutations were generated for, per file: every line a mutation
48
+ # changes, the same lines the line-range filter judged it by. Taken before
49
+ # any later filter: a line whose mutations were all disabled or judged
50
+ # equivalent was still reached by an operator.
48
51
  def mutated_lines(mutations)
49
- mutations.group_by(&:file_path).transform_values { |group| group.map(&:line).uniq.sort }
52
+ mutations.group_by(&:file_path).transform_values do |group|
53
+ group.flat_map { |mutation| mutation.changed_lines.to_a }.uniq.sort
54
+ end
50
55
  end
51
56
 
52
57
  def compute_disabled_mutations(disabled_filter)
@@ -60,11 +65,12 @@ class Evilution::Runner::MutationPlanner
60
65
  generation.skipped + disabled_filter.disabled.length + sig_filter.skipped
61
66
  end
62
67
 
63
- def build_plan(equivalent_filter, disabled_mutations, skipped_count)
68
+ def build_plan(equivalent_filter, disabled_mutations, skipped_count, skipped_sites)
64
69
  Plan.new(
65
70
  enabled: equivalent_filter.enabled,
66
71
  equivalent: equivalent_filter.equivalent,
67
72
  skipped_count: skipped_count,
73
+ skipped_sites: skipped_sites,
68
74
  disabled_mutations: disabled_mutations
69
75
  )
70
76
  end
@@ -85,25 +91,27 @@ class Evilution::Runner::MutationPlanner
85
91
  seen.values
86
92
  end
87
93
 
94
+ # Each file is read once, and its every subject is mutated from that one
95
+ # source: the mutations of a file then share it as their original source.
88
96
  def generate(subjects)
89
- filter = build_ignore_filter
97
+ filter = Evilution::AST::SkipFilter.for(config)
90
98
  operator_options = build_operator_options
99
+ file_sources = Hash.new { |sources, path| sources[path] = File.read(path) }
91
100
  mutations = subjects.flat_map do |subject|
92
- registry.mutations_for(subject, filter: filter, operator_options: operator_options)
101
+ registry.mutations_for(subject, filter: filter, operator_options: operator_options, file_sources: file_sources)
93
102
  end
94
- skipped = filter ? filter.skipped_count : 0
95
- GenerationResult.new(mutations: mutations, skipped: skipped)
96
- end
103
+ return GenerationResult.new(mutations: mutations, skipped: 0, skipped_sites: {}) unless filter
97
104
 
98
- def build_operator_options
99
- { skip_heredoc_literals: config.skip_heredoc_literals? }
105
+ GenerationResult.new(mutations: mutations, skipped: filter.skipped_count(made: mutations),
106
+ skipped_sites: filter.skipped_by_site(made: mutations))
100
107
  end
101
108
 
102
- def build_ignore_filter
103
- patterns = config.ignore_patterns
104
- return nil if patterns.nil? || patterns.empty?
105
-
106
- Evilution::AST::Pattern::Filter.new(patterns)
109
+ # The operators build no mutation of lines outside a requested line range:
110
+ # those are out of the run's scope, as the subjects the range leaves out
111
+ # are, not suppressed within it -- so they are neither skipped nor reported
112
+ # as mutated lines.
113
+ def build_operator_options
114
+ { skip_heredoc_literals: config.skip_heredoc_literals?, line_ranges: config.line_ranges }
107
115
  end
108
116
 
109
117
  def filter_disabled(mutations)
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../runner"
4
+
5
+ # Wall time of each phase of a run, in the order the phases ran. A phase
6
+ # measured more than once adds up.
7
+ class Evilution::Runner::PhaseClock
8
+ def initialize
9
+ @phases = {}
10
+ end
11
+
12
+ def measure(phase)
13
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
14
+ yield
15
+ ensure
16
+ @phases[phase] = @phases.fetch(phase, 0.0) + (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
17
+ end
18
+
19
+ def to_h
20
+ @phases.dup.freeze
21
+ end
22
+ end
@@ -67,7 +67,7 @@ class Evilution::Runner::ReportPublisher
67
67
  when :json
68
68
  Evilution::Reporter::JSON.new(integration: config.integration)
69
69
  when :text
70
- Evilution::Reporter::CLI.new(min_score: config.min_score)
70
+ Evilution::Reporter::CLI.new(min_score: config.min_score, since: config.since)
71
71
  when :html
72
72
  Evilution::Reporter::HTML.new(baseline: load_baseline_session, integration: config.integration)
73
73
  end
@@ -5,6 +5,7 @@ require_relative "../ast/inheritance_scanner"
5
5
  require_relative "../ast/uncovered_code"
6
6
  require_relative "../diagnostic"
7
7
  require_relative "../git/changed_files"
8
+ require_relative "../mutator/line_range_filter"
8
9
 
9
10
  class Evilution::Runner::SubjectPipeline
10
11
  autoload :Target, File.expand_path("subject_pipeline/target", __dir__)
@@ -29,6 +30,13 @@ class Evilution::Runner::SubjectPipeline
29
30
  @target_files ||= resolve_target_files
30
31
  end
31
32
 
33
+ # Every subject #call parsed, before the target and line ranges filtered
34
+ # them; empty before #call and once #report_uncovered_code is done with
35
+ # them.
36
+ def parsed_subjects
37
+ (@subjects_by_file || {}).values.flatten
38
+ end
39
+
32
40
  # Warns about the targeted lines that hold code outside every subject and
33
41
  # that no mutation reached either. mutated_lines: the lines mutations were
34
42
  # generated for, per file -- known only once #call's subjects are planned.
@@ -77,8 +85,9 @@ class Evilution::Runner::SubjectPipeline
77
85
 
78
86
  def warn_uncovered(entry)
79
87
  Evilution::Diagnostic.warn(
80
- "[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject " \
81
- "(class-body code such as DSL calls is not mutated); no mutations target those lines."
88
+ "[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject; " \
89
+ "no mutations target those lines. Of the DSL calls in a class body, only scopes, aasm machines, and " \
90
+ "callback or validation declarations with options or an inline block or lambda are mutated."
82
91
  )
83
92
  end
84
93
 
@@ -125,9 +134,12 @@ class Evilution::Runner::SubjectPipeline
125
134
  matched
126
135
  end
127
136
 
137
+ # With --since the target files are the changed ones, and none having
138
+ # changed means there is nothing to mutate, not that the git-changed files
139
+ # stand in.
128
140
  def resolve_target_files
129
141
  return resolve_source_glob if target.source_glob?
130
- return config.target_files unless config.target_files.empty?
142
+ return config.target_files unless config.target_files.empty? && !config.since?
131
143
 
132
144
  @used_git_fallback = true
133
145
  Evilution::Git::ChangedFiles.new.call
@@ -135,6 +147,7 @@ class Evilution::Runner::SubjectPipeline
135
147
 
136
148
  def build_no_match_error
137
149
  base = "no subject matched '#{target}'"
150
+ return "#{base} in the files changed since #{config.since}" if config.since?
138
151
  return base unless @used_git_fallback
139
152
 
140
153
  "#{base}; scanned git-changed files only. Pass file paths or " \
@@ -142,13 +155,10 @@ class Evilution::Runner::SubjectPipeline
142
155
  end
143
156
 
144
157
  def filter_by_line_ranges(subjects)
158
+ filter = Evilution::Mutator::LineRangeFilter.new(config.line_ranges)
145
159
  subjects.select do |subject|
146
- range = config.line_ranges[subject.file_path]
147
- next true unless range
148
-
149
160
  subject_start = subject.line_number
150
- subject_end = subject_start + subject.source.count("\n")
151
- subject_start <= range.last && subject_end >= range.first
161
+ filter.cover?(subject.file_path, subject_start..(subject_start + subject.source.count("\n")))
152
162
  end
153
163
  end
154
164
  end
@@ -18,44 +18,31 @@ class Evilution::Runner
18
18
  @hooks = hooks
19
19
  @parser = Evilution::AST::Parser.new
20
20
  @registry = Evilution::Mutator::Registry.for_profile(config.profile)
21
- @cache = config.incremental? ? Evilution::Cache.new : nil
21
+ @cache = config.incremental? ? Evilution::Cache.new(keys: Evilution::Cache::Keys.for(config)) : nil
22
22
  end
23
23
 
24
24
  def call
25
25
  install_signal_handlers
26
26
  configure_child_output
27
- emit_parallel_db_warning
28
27
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
28
+ clock = Evilution::Runner::PhaseClock.new
29
29
 
30
- subjects = subject_pipeline.call
30
+ subjects = clock.measure(:subject_discovery) { discover_subjects }
31
31
  log_memory("after parse_subjects", "#{subjects.length} subjects")
32
32
 
33
- perform_preload
33
+ clock.measure(:preload) { perform_preload }
34
34
  log_memory("after preload") if rails_root_detected?
35
+ emit_parallel_db_warning
35
36
 
36
- run_canary
37
-
38
- baseline_result = run_baseline(subjects)
39
-
40
- plan = mutation_planner.call(subjects)
41
- subject_pipeline.report_uncovered_code(plan.mutated_lines)
42
- release_subject_nodes(subjects)
43
- clear_operator_caches
44
- execution = run_mutations(plan.enabled, baseline_result)
45
- results = execution.results + equivalent_results(plan.equivalent)
46
- log_memory("after run_mutations", "#{results.length} results")
37
+ clock.measure(:canary) { run_canary }
38
+ baseline_result = clock.measure(:baseline) { run_baseline(subjects) }
39
+ plan = clock.measure(:planning) { plan_mutations(subjects) }
40
+ execution = clock.measure(:execution) { run_mutations(plan.enabled, baseline_result) }
41
+ outcome = clock.measure(:finalize) { finalize(plan, execution, subjects) }
47
42
 
48
43
  duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
49
-
50
- audited_files = subjects.map(&:file_path).uniq.sort
51
- summary = Evilution::Result::Summary.new(results: results, duration: duration, truncated: execution.truncated,
52
- skipped: plan.skipped_count,
53
- disabled_mutations: plan.disabled_mutations,
54
- unresolved_target_files: target_spec_audit.call(audited_files),
55
- target_file_count: audited_files.length,
56
- infra_retried: execution.infra_retried,
57
- uncovered_code: subject_pipeline.uncovered_code,
58
- baseline_failures: baseline_result ? baseline_result.failures : [])
44
+ summary = build_summary(outcome: outcome, duration: duration, plan: plan, execution: execution,
45
+ baseline_result: baseline_result, phases: clock.to_h)
59
46
  output_report(summary)
60
47
  save_session(summary)
61
48
 
@@ -74,6 +61,14 @@ class Evilution::Runner
74
61
  @subject_pipeline ||= Evilution::Runner::SubjectPipeline.new(config, parser: parser, registry: registry)
75
62
  end
76
63
 
64
+ # The cache keys a subject apart from the rest of its file, which takes
65
+ # every subject of the file, whether this run mutates it or not.
66
+ def discover_subjects
67
+ subjects = subject_pipeline.call
68
+ cache.index(subject_pipeline.parsed_subjects) if cache
69
+ subjects
70
+ end
71
+
77
72
  def mutation_planner
78
73
  @mutation_planner ||= Evilution::Runner::MutationPlanner.new(config, registry: registry)
79
74
  end
@@ -98,8 +93,64 @@ class Evilution::Runner
98
93
  isolation_resolver.rails_root_detected?
99
94
  end
100
95
 
96
+ # With coverage targeting, the lines the preload runs are counted: the
97
+ # coverage map is built in a child forked after it, and needs the preloaded
98
+ # files, and what ran while they loaded, to tell which lines no test executes.
99
+ # Only when the tests run in forked children, which hand the counting back
100
+ # before they run (Evilution::Coverage::LoadTracker.release_inherited): a
101
+ # test run in this process would find it paused, and could not start it.
101
102
  def perform_preload
102
- isolation_resolver.perform_preload
103
+ tracker = Evilution::Coverage::LoadTracker.new
104
+ isolation_resolver.perform_preload { tracker.start if count_preload? }
105
+ ensure
106
+ tracker.pause if tracker
107
+ end
108
+
109
+ def count_preload?
110
+ config.coverage_targeting? && isolation_resolver.fork_isolation?
111
+ end
112
+
113
+ def plan_mutations(subjects)
114
+ plan = mutation_planner.call(subjects)
115
+ subject_pipeline.report_uncovered_code(plan.mutated_lines)
116
+ release_subject_nodes(subjects)
117
+ clear_operator_caches
118
+ plan
119
+ end
120
+
121
+ # What the summary needs once the mutations have run: every result, with
122
+ # the equivalent mutations folded in, and the audit of the target files.
123
+ Outcome = Data.define(:results, :target_file_count, :unresolved_target_files)
124
+ private_constant :Outcome
125
+
126
+ def finalize(plan, execution, subjects)
127
+ results = execution.results + equivalent_results(plan.equivalent)
128
+ log_memory("after run_mutations", "#{results.length} results")
129
+ audited_files = subjects.map(&:file_path).uniq.sort
130
+ Outcome.new(results: results, target_file_count: audited_files.length,
131
+ unresolved_target_files: target_spec_audit.call(audited_files))
132
+ end
133
+
134
+ def build_summary(outcome:, duration:, plan:, execution:, baseline_result:, phases:)
135
+ Evilution::Result::Summary.new(
136
+ results: outcome.results, duration: duration, truncated: execution.truncated,
137
+ skipped: plan.skipped_count, skipped_sites: plan.skipped_sites,
138
+ disabled_mutations: plan.disabled_mutations,
139
+ unresolved_target_files: outcome.unresolved_target_files,
140
+ target_file_count: outcome.target_file_count,
141
+ infra_retried: execution.infra_retried,
142
+ uncovered_code: subject_pipeline.uncovered_code,
143
+ baseline_failures: baseline_result ? baseline_result.failures : [],
144
+ timings: run_timings(phases, baseline_result, execution)
145
+ )
146
+ end
147
+
148
+ def run_timings(phases, baseline_result, execution)
149
+ Evilution::Result::RunTimings.new(
150
+ phases: phases,
151
+ baseline_spec_files: baseline_result ? baseline_result.spec_file_durations : {},
152
+ workers: execution.worker_stats
153
+ )
103
154
  end
104
155
 
105
156
  def release_subject_nodes(subjects)
@@ -156,7 +207,14 @@ class Evilution::Runner
156
207
  mutation_executor.call(mutations, baseline_result)
157
208
  end
158
209
 
210
+ # Workers forked from a preloaded app get copies of a shared SQLite
211
+ # database, so the advice to give each worker its own is only for runs
212
+ # whose workers cannot: copies off, no preloaded app, or a preloaded
213
+ # connection the copies leave alone (a file: URI, one set up outside
214
+ # database.yml).
159
215
  def emit_parallel_db_warning
216
+ return if config.copy_worker_databases? && Evilution::WorkerDatabase.copies_cover?
217
+
160
218
  Evilution::ParallelDbWarning.warn_if_sqlite_parallel(config)
161
219
  end
162
220
 
@@ -242,6 +300,7 @@ require_relative "git/changed_files"
242
300
  require_relative "result/mutation_result"
243
301
  require_relative "result/summary"
244
302
  require_relative "baseline"
303
+ require_relative "coverage/load_tracker"
245
304
  require_relative "cache"
246
305
  require_relative "parallel/pool"
247
306
  require_relative "process_supervisor"
@@ -249,6 +308,7 @@ require_relative "session/store"
249
308
  require_relative "temp_dir_tracker"
250
309
  require_relative "rails_detector"
251
310
  require_relative "parallel_db_warning"
311
+ require_relative "worker_database"
252
312
  require_relative "child_output"
253
313
  require_relative "runner/subject_pipeline"
254
314
  require_relative "runner/target_spec_audit"
@@ -258,3 +318,4 @@ require_relative "runner/baseline_runner"
258
318
  require_relative "runner/diagnostics"
259
319
  require_relative "runner/mutation_executor"
260
320
  require_relative "runner/report_publisher"
321
+ require_relative "runner/phase_clock"
@@ -36,8 +36,8 @@ class Evilution::Session::Diff
36
36
  end
37
37
 
38
38
  def call(base_data, head_data)
39
- base_survivors = base_data["survived"] || []
40
- head_survivors = head_data["survived"] || []
39
+ base_survivors = untested(base_data)
40
+ head_survivors = untested(head_data)
41
41
  fixed, new_survivors, persistent = partition_survivors(base_survivors, head_survivors)
42
42
 
43
43
  Result.new(
@@ -50,6 +50,13 @@ class Evilution::Session::Diff
50
50
 
51
51
  private
52
52
 
53
+ # A mutation no test executes is a survivor that was not run, so it sits with
54
+ # the survivors here: one that survived before is not fixed by now being
55
+ # reported uncovered.
56
+ def untested(data)
57
+ (data["survived"] || []) + (data["uncovered"] || [])
58
+ end
59
+
53
60
  def partition_survivors(base_survivors, head_survivors)
54
61
  base_keys = base_survivors.to_set { |m| mutation_key(m) }
55
62
  head_keys = head_survivors.to_set { |m| mutation_key(m) }
@@ -69,12 +69,20 @@ class Evilution::Session::Store
69
69
  git: git_context,
70
70
  summary: build_summary(summary),
71
71
  survived: summary.survived_results.map { |r| build_mutation_detail(r) },
72
+ uncovered: summary.uncovered_results.map { |r| build_mutation_detail(r) },
72
73
  coverage_gaps: build_coverage_gaps(summary),
74
+ **status_counts(summary)
75
+ }
76
+ end
77
+
78
+ def status_counts(summary)
79
+ {
73
80
  killed_count: summary.killed,
74
81
  timed_out_count: summary.timed_out,
75
82
  error_count: summary.errors,
76
83
  neutral_count: summary.neutral,
77
84
  equivalent_count: summary.equivalent,
85
+ uncovered_count: summary.uncovered,
78
86
  skipped_count: summary.skipped
79
87
  }
80
88
  end
@@ -88,11 +96,18 @@ class Evilution::Session::Store
88
96
  errors: summary.errors,
89
97
  neutral: summary.neutral,
90
98
  equivalent: summary.equivalent,
99
+ uncovered: summary.uncovered,
91
100
  score: summary.score.round(4),
92
101
  duration: summary.duration.round(4)
93
102
  }
94
- data[:skipped] = summary.skipped if summary.skipped.positive?
95
- data
103
+ data.merge(skip_fields(summary))
104
+ end
105
+
106
+ def skip_fields(summary)
107
+ fields = {}
108
+ fields[:skipped] = summary.skipped if summary.skipped.positive?
109
+ fields[:skipped_sites] = summary.skipped_sites unless summary.skipped_sites.empty?
110
+ fields
96
111
  end
97
112
 
98
113
  def build_mutation_detail(result)
@@ -31,6 +31,16 @@ class Evilution::SourceAstCache
31
31
  result
32
32
  end
33
33
 
34
+ # Parses and keeps a source ahead of the first #fetch for it, but only while
35
+ # there is room: an entry that pushed out an earlier one would cost the
36
+ # parse it is meant to save.
37
+ def preload(source)
38
+ return if @entries.length >= @max_entries
39
+
40
+ key = Digest::SHA256.digest(source)
41
+ @entries[key] = Prism.parse(source) unless @entries.key?(key)
42
+ end
43
+
34
44
  private
35
45
 
36
46
  def evict_until_within_bounds
@@ -40,8 +40,23 @@ class Evilution::SpecAstCache
40
40
  @entries.key?(path)
41
41
  end
42
42
 
43
+ # Parses and keeps a file ahead of the first #fetch for it, but only where
44
+ # it fits beside what is already held: an entry that pushed out an earlier
45
+ # one would cost the parse it is meant to save. A file that does not parse
46
+ # raises as #fetch would.
47
+ def preload(path)
48
+ return if cached?(path) || full?
49
+
50
+ blocks = parse(path)
51
+ insert(path, blocks) if @total_blocks + blocks.length <= @max_blocks
52
+ end
53
+
43
54
  private
44
55
 
56
+ def full?
57
+ @entries.length >= @max_files || @total_blocks >= @max_blocks
58
+ end
59
+
45
60
  def insert(path, blocks)
46
61
  @entries[path] = blocks
47
62
  @total_blocks += blocks.length
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prism"
4
+ require_relative "../spec_resolver"
5
+
6
+ # The constants a Ruby file refers to, read from Prism's token stream rather
7
+ # than from raw text: a name inside a comment, a string, a heredoc or an
8
+ # embedded document is not a reference, while one inside string interpolation
9
+ # is. Lexing needs no parse tree, so the cost stays close to a regex scan.
10
+ class Evilution::SpecResolver::ConstantReferences
11
+ # Tokens after which a `describe` identifier is not a call to describe.
12
+ NOT_A_CALL = %i[SYMBOL_BEGIN KEYWORD_DEF].freeze
13
+ RECEIVER_SEPARATORS = %i[DOT AMPERSAND_DOT].freeze
14
+ OPENING_PARENTHESES = %i[PARENTHESIS_LEFT PARENTHESIS_LEFT_PARENTHESES].freeze
15
+
16
+ def initialize(text)
17
+ @tokens = Prism.lex(text).value.map(&:first)
18
+ end
19
+
20
+ # Every constant segment, so Timeline::NightWarnings::LIMIT yields all three.
21
+ # A symbol spelled like a constant (:Round) is not a reference.
22
+ def constants
23
+ [nil, *@tokens].each_cons(2).filter_map do |previous, token|
24
+ token.value if token.type == :CONSTANT && !(previous && previous.type == :SYMBOL_BEGIN)
25
+ end
26
+ end
27
+
28
+ # The last segment of the constant passed to the file's first `describe`
29
+ # (bare or `RSpec.`), or nil when there is none or it is given anything else,
30
+ # such as a string. Later, nested describes never count.
31
+ def describe_constant
32
+ index = @tokens.each_index.find { |i| describe_call?(i) }
33
+ index && constant_path_after(index + 1)
34
+ end
35
+
36
+ private
37
+
38
+ def describe_call?(index)
39
+ token = @tokens[index]
40
+ return false unless token.type == :IDENTIFIER && token.value == "describe"
41
+ return true if index.zero?
42
+
43
+ previous = @tokens[index - 1].type
44
+ return rspec?(@tokens[index - 2]) if RECEIVER_SEPARATORS.include?(previous)
45
+
46
+ !NOT_A_CALL.include?(previous)
47
+ end
48
+
49
+ def rspec?(token)
50
+ token.type == :CONSTANT && token.value == "RSpec"
51
+ end
52
+
53
+ # The token stream always ends with EOF, so lookahead never runs off the end.
54
+ def constant_path_after(index)
55
+ index += 1 if OPENING_PARENTHESES.include?(@tokens[index].type)
56
+ index += 1 if @tokens[index].type == :UCOLON_COLON
57
+ last = nil
58
+ while @tokens[index].type == :CONSTANT
59
+ last = @tokens[index].value
60
+ break unless @tokens[index + 1].type == :COLON_COLON
61
+
62
+ index += 2
63
+ end
64
+ last
65
+ end
66
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../spec_resolver"
4
+
5
+ # Project-relative filesystem lookups shared by the spec resolver and its
6
+ # collaborators. Each lookup succeeds against the current CWD; when the caller
7
+ # is an isolated worker chdir'd into a per-mutation sandbox (signalled by
8
+ # Evilution.in_isolated_worker?), it also tries PROJECT_ROOT so the sandbox CWD
9
+ # does not break resolution. The PROJECT_ROOT fallback stays gated on that flag
10
+ # so a caller that chdirs on purpose (e.g. a fixture project) never resolves
11
+ # into the evilution dev tree.
12
+ module Evilution::SpecResolver::ProjectFiles
13
+ module_function
14
+
15
+ def exist?(path)
16
+ return true if File.exist?(path)
17
+ return false unless Evilution.in_isolated_worker?
18
+
19
+ File.exist?(File.expand_path(path, Evilution::PROJECT_ROOT))
20
+ end
21
+
22
+ def directory?(path)
23
+ return true if File.directory?(path)
24
+ return false unless Evilution.in_isolated_worker?
25
+
26
+ File.directory?(File.expand_path(path, Evilution::PROJECT_ROOT))
27
+ end
28
+
29
+ # base: PROJECT_ROOT yields paths relative to the root, matching the shape
30
+ # of a CWD glob.
31
+ def glob(pattern)
32
+ return Dir.glob(pattern) unless Evilution.in_isolated_worker?
33
+
34
+ Dir.glob(pattern, base: Evilution::PROJECT_ROOT)
35
+ end
36
+
37
+ # Raw bytes of the file, or "" when it is missing or unreadable. Binary so a
38
+ # file with invalid UTF-8 cannot raise during an ASCII-only regex scan.
39
+ def read(path)
40
+ File.binread(locate(path))
41
+ rescue SystemCallError
42
+ ""
43
+ end
44
+
45
+ def locate(path)
46
+ return path if File.exist?(path) || !Evilution.in_isolated_worker?
47
+
48
+ File.expand_path(path, Evilution::PROJECT_ROOT)
49
+ end
50
+ end