evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../validators"
4
+ require_relative "base"
5
+
6
+ # The git ref given as --since, stripped of the space around it; nil when none
7
+ # was given. The commands check it before git is asked about the ref, so that
8
+ # an empty ref or a conflicting target is reported as such rather than as a
9
+ # ref git does not know, or as a missing repository.
10
+ class Evilution::Config::Validators::Since < Evilution::Config::Validators::Base
11
+ def self.call(value, target:)
12
+ return nil if value.nil?
13
+
14
+ ref = value.to_s.strip
15
+ raise Evilution::ConfigError, "--since needs a git ref, such as main or origin/main" if ref.empty?
16
+ return ref unless target.to_s.start_with?("source:")
17
+
18
+ raise Evilution::ConfigError,
19
+ "--since picks the changed files itself and cannot be combined with --target source:GLOB; " \
20
+ "pass the files to limit it to instead"
21
+ end
22
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../validators"
4
+ require_relative "base"
5
+ require_relative "../../ast/skip_sites/presets"
6
+
7
+ # `skip_sites` names the presets of low-value code to leave unmutated (see
8
+ # AST::SkipSites::Presets). `false` is accepted as none.
9
+ class Evilution::Config::Validators::SkipSites < Evilution::Config::Validators::Base
10
+ def self.call(value)
11
+ return [] if value == false
12
+
13
+ Array(value).map { |name| preset_name!(name) }.uniq
14
+ end
15
+
16
+ def self.preset_name!(name)
17
+ known = Evilution::AST::SkipSites::Presets.names
18
+ symbol = name.to_sym if name.is_a?(String) || name.is_a?(Symbol)
19
+ return symbol if known.include?(symbol)
20
+
21
+ raise Evilution::ConfigError, "skip_sites: unknown preset #{name.inspect} (known: #{known.join(", ")})"
22
+ end
23
+ private_class_method :preset_name!
24
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../validators"
4
+ require_relative "base"
5
+
6
+ class Evilution::Config::Validators::TimeoutCeiling < Evilution::Config::Validators::Base
7
+ def self.call(value)
8
+ coerce_positive_number!(value, name: "timeout_ceiling")
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../validators"
4
+ require_relative "base"
5
+
6
+ class Evilution::Config::Validators::TimeoutFactor < Evilution::Config::Validators::Base
7
+ def self.call(value)
8
+ coerce_positive_number!(value, name: "timeout_factor")
9
+ end
10
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../validators"
4
+ require_relative "base"
5
+
6
+ class Evilution::Config::Validators::TimeoutFloor < Evilution::Config::Validators::Base
7
+ def self.call(value)
8
+ coerce_positive_number!(value, name: "timeout_floor")
9
+ end
10
+ end
@@ -21,7 +21,9 @@ class Evilution::Config
21
21
  example_targeting_fallback: :full_file, example_targeting_strategy: :lexical,
22
22
  example_targeting_cache: { max_files: 50, max_blocks: 10_000 },
23
23
  quiet_children: false, quiet_children_dir: "tmp/evilution_children",
24
- profile: :default, canary: true, warmup: :none
24
+ profile: :default, canary: true, warmup: :none, stop_on_kill: true, killer_first: false,
25
+ timeout_factor: 10, timeout_floor: 10, timeout_ceiling: 60, skip_uncovered: true,
26
+ probe_first: false, skip_sites: [], copy_worker_databases: true
25
27
  }.freeze
26
28
 
27
29
  attr_reader :target_files, :schema_version, :timeout, :format,
@@ -33,12 +35,19 @@ class Evilution::Config
33
35
  :fallback_to_full_suite, :preload, :spec_mappings, :spec_pattern,
34
36
  :example_targeting, :example_targeting_fallback, :example_targeting_strategy,
35
37
  :example_targeting_cache,
36
- :spec_selector, :quiet_children, :quiet_children_dir, :profile, :canary, :warmup
38
+ :spec_selector, :quiet_children, :quiet_children_dir, :profile, :canary, :warmup,
39
+ :stop_on_kill, :killer_first, :timeout_factor, :timeout_floor, :timeout_ceiling, :skip_uncovered,
40
+ :probe_first, :skip_sites, :copy_worker_databases, :since
37
41
 
42
+ # since: the git ref whose changed lines alone the run mutates. Taken only
43
+ # from the caller, like the target files it stands in for: a config file
44
+ # naming one would scope every run, an explicit file list included.
38
45
  def initialize(**options)
39
46
  skip_file = options.delete(:skip_config_file) ? true : false
47
+ since = options.delete(:since)
40
48
  merged = Sources.merge(explicit: options, skip_file: skip_file)
41
49
  assign_attributes(merged)
50
+ assign_since(since)
42
51
  freeze
43
52
  end
44
53
 
@@ -62,6 +71,10 @@ class Evilution::Config
62
71
  !target.nil?
63
72
  end
64
73
 
74
+ def since?
75
+ !since.nil?
76
+ end
77
+
65
78
  def fail_fast?
66
79
  !fail_fast.nil?
67
80
  end
@@ -74,6 +87,22 @@ class Evilution::Config
74
87
  canary
75
88
  end
76
89
 
90
+ def stop_on_kill?
91
+ stop_on_kill
92
+ end
93
+
94
+ # Whether a run that stops at its first kill tries the examples most likely
95
+ # to kill first. Only such a run gains anything from the order.
96
+ def killer_first?
97
+ killer_first && stop_on_kill?
98
+ end
99
+
100
+ # Whether each mutation's limit follows the baseline time of its spec files.
101
+ # A timeout given on the command line or in the config file keeps winning.
102
+ def adaptive_timeout?
103
+ @adaptive_timeout
104
+ end
105
+
77
106
  def incremental?
78
107
  incremental
79
108
  end
@@ -114,6 +143,24 @@ class Evilution::Config
114
143
  fallback_to_full_suite
115
144
  end
116
145
 
146
+ # Whether a mutation no test executes is reported uncovered without being
147
+ # run. Only coverage targeting builds the coverage map that can tell.
148
+ def skip_uncovered?
149
+ skip_uncovered && coverage_targeting?
150
+ end
151
+
152
+ # Whether each subject's sentinel runs first and, when it survives, implies
153
+ # its subject's other mutations survive too, without running them.
154
+ def probe_first?
155
+ probe_first
156
+ end
157
+
158
+ # Whether a parallel worker that would share the preloaded SQLite test
159
+ # database with the others gets a copy of its own.
160
+ def copy_worker_databases?
161
+ copy_worker_databases
162
+ end
163
+
117
164
  def self.file_options
118
165
  FileLoader.load
119
166
  end
@@ -130,8 +177,14 @@ class Evilution::Config
130
177
  # Omit to keep the legacy lenient behavior (unknown keys ignored).
131
178
  schema_version: #{CURRENT_SCHEMA_VERSION}
132
179
 
133
- # Per-mutation timeout in seconds (default: 30)
180
+ # Per-mutation timeout in seconds. Left unset, each mutation's limit is
181
+ # timeout_factor x the baseline time of its spec files, kept between
182
+ # timeout_floor and timeout_ceiling; 30 is used where the baseline has no
183
+ # time for them. Setting it gives every mutation this fixed limit.
134
184
  # timeout: 30
185
+ # timeout_factor: 10
186
+ # timeout_floor: 10
187
+ # timeout_ceiling: 60
135
188
 
136
189
  # Output format: text or json (default: text)
137
190
  # format: text
@@ -144,12 +197,28 @@ class Evilution::Config
144
197
  # (default: true). Set false to skip (e.g. for CI speed).
145
198
  # canary: true
146
199
 
200
+ # Stop each mutation's test run at the first failure that decides the
201
+ # verdict (default: true). Set false to run every selected example.
202
+ # stop_on_kill: true
203
+
204
+ # With RSpec and stop_on_kill, run first the examples that killed other
205
+ # mutations nearby, then the cheapest (default: false). Only for a suite
206
+ # whose examples pass in any order: one that leans on the order they are
207
+ # defined in reports kills that a run in that order would not.
208
+ # killer_first: false
209
+
147
210
  # Test integration: rspec, minitest, test_unit (default: rspec)
148
211
  # integration: rspec
149
212
 
150
213
  # Number of parallel workers (default: 1)
151
214
  # jobs: 1
152
215
 
216
+ # With jobs > 1 and a preloaded Rails app, give each worker its own copy
217
+ # of a SQLite test database the workers would otherwise share (one
218
+ # database.yml path for all, or a TEST_ENV_NUMBER database that is not
219
+ # prepared). Set false to let them share it (default: true).
220
+ # copy_worker_databases: true
221
+
153
222
  # Stop after N surviving mutants (default: disabled)
154
223
  # fail_fast: 1
155
224
 
@@ -199,6 +268,17 @@ class Evilution::Config
199
268
  # any file the map has not fully built)
200
269
  # example_targeting_strategy: lexical
201
270
 
271
+ # With coverage targeting, report a mutation as `uncovered`, without
272
+ # running it, when no example executes any line it changes (default: true). It scores as a
273
+ # survivor. Lines that run only while files load (class bodies, constants,
274
+ # `def` lines) count as executed. Set false to run them anyway.
275
+ # skip_uncovered: true
276
+
277
+ # Experimental. Run each method's whole-body `raise` mutation first; when
278
+ # it survives, report the method's other mutations as survivors implied
279
+ # by it, without running them (default: false).
280
+ # probe_first: false
281
+
202
282
  # Behavior when targeting finds no matching example (default: full_file).
203
283
  # full_file - run every example in the resolved spec files
204
284
  # unresolved - mark the mutation :unresolved and skip
@@ -210,11 +290,23 @@ class Evilution::Config
210
290
  # max_blocks: 10000
211
291
 
212
292
  # AST patterns to skip during mutation generation (default: [])
213
- # See docs/ast_pattern_syntax.md for pattern syntax
293
+ # See docs/ast_pattern_syntax.md for pattern syntax. A pattern skips every
294
+ # mutation inside the node it matches: a call pattern covers the call's
295
+ # receiver, arguments and block.
214
296
  # ignore_patterns:
215
297
  # - "call{name=info, receiver=call{name=logger}}"
216
298
  # - "call{name=debug|warn}"
217
299
 
300
+ # Presets of low-value code to leave unmutated (default: none). Each
301
+ # skips only mutations that change what the code reports, never the
302
+ # raise, the raised class or the condition around it:
303
+ # logging - log lines and deprecation warnings
304
+ # debug_output - puts/print/warn/p/pp and writes to $stdout/$stderr
305
+ # error_messages - the message of a raised error
306
+ # requires - require/require_relative run while the file loads
307
+ # (no subject mutates these yet, so it skips nothing)
308
+ # skip_sites: [logging, error_messages]
309
+
218
310
  # Operator profile: default or strict (default: default).
219
311
  # strict adds aggressive mutators (e.g. replaces `x.predicate?` with
220
312
  # `nil`, or appends `rescue nil` to a raising call) intended for
@@ -229,6 +321,7 @@ class Evilution::Config
229
321
  assign_simple_attributes(merged)
230
322
  assign_validated_attributes(merged)
231
323
  assign_example_targeting(merged)
324
+ assign_adaptive_timeout(merged)
232
325
  @spec_selector = Builders::SpecSelector.call(
233
326
  spec_files: @spec_files,
234
327
  spec_mappings: @spec_mappings,
@@ -261,7 +354,12 @@ class Evilution::Config
261
354
  fallback_to_full_suite: nil,
262
355
  quiet_children: nil,
263
356
  quiet_children_dir: nil,
264
- canary: nil
357
+ canary: nil,
358
+ stop_on_kill: nil,
359
+ killer_first: nil,
360
+ skip_uncovered: nil,
361
+ probe_first: nil,
362
+ copy_worker_databases: nil
265
363
  }.freeze
266
364
  private_constant :SIMPLE_ATTR_TRANSFORMS
267
365
 
@@ -274,8 +372,9 @@ class Evilution::Config
274
372
  end
275
373
 
276
374
  VALIDATED_ATTRS = %i[
277
- integration jobs fail_fast isolation ignore_patterns
375
+ integration jobs fail_fast isolation ignore_patterns skip_sites
278
376
  hooks preload spec_mappings spec_pattern profile warmup
377
+ timeout_factor timeout_floor timeout_ceiling
279
378
  ].freeze
280
379
  private_constant :VALIDATED_ATTRS
281
380
 
@@ -292,6 +391,18 @@ class Evilution::Config
292
391
  @example_targeting_strategy = Validators::ExampleTargetingStrategy.call(merged[:example_targeting_strategy])
293
392
  @example_targeting_cache = Validators::ExampleTargetingCache.call(merged[:example_targeting_cache])
294
393
  end
394
+
395
+ def assign_since(since)
396
+ @since = Validators::Since.call(since, target: target)
397
+ end
398
+
399
+ def assign_adaptive_timeout(merged)
400
+ @adaptive_timeout = !merged[:timeout_given]
401
+ return if @timeout_floor <= @timeout_ceiling
402
+
403
+ raise Evilution::ConfigError,
404
+ "timeout_floor (#{@timeout_floor}) must not exceed timeout_ceiling (#{@timeout_ceiling})"
405
+ end
295
406
  end
296
407
 
297
408
  require_relative "config/file_loader"
@@ -306,6 +417,7 @@ require_relative "config/validators/fail_fast"
306
417
  require_relative "config/validators/preload"
307
418
  require_relative "config/validators/hooks"
308
419
  require_relative "config/validators/ignore_patterns"
420
+ require_relative "config/validators/skip_sites"
309
421
  require_relative "config/validators/spec_pattern"
310
422
  require_relative "config/validators/spec_mappings"
311
423
  require_relative "config/validators/example_targeting_fallback"
@@ -313,6 +425,10 @@ require_relative "config/validators/example_targeting_strategy"
313
425
  require_relative "config/validators/example_targeting_cache"
314
426
  require_relative "config/validators/profile"
315
427
  require_relative "config/validators/warmup"
428
+ require_relative "config/validators/since"
429
+ require_relative "config/validators/timeout_factor"
430
+ require_relative "config/validators/timeout_floor"
431
+ require_relative "config/validators/timeout_ceiling"
316
432
  require_relative "config/builders"
317
433
  require_relative "config/builders/spec_resolver"
318
434
  require_relative "config/builders/spec_selector"
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "coverage"
4
+ require_relative "../coverage"
5
+
6
+ # Counts lines from before the project is preloaded until it has been. Ruby
7
+ # counts only the files loaded after counting is set up, so without this a
8
+ # preloaded file is missing from the coverage map altogether -- and the map
9
+ # needs to know what ran while the files loaded (a class body, a `def`, a
10
+ # constant) to tell a line no example executes from one that runs only at load.
11
+ #
12
+ # Paused afterwards: the coverage map's build resumes it in a child of its own
13
+ # (Evilution::Coverage::MapBuilder). Every other child forked from this process
14
+ # runs tests and must not inherit it paused: set up, coverage cannot be started
15
+ # again, so a spec helper that starts it (as SimpleCov does) would raise there.
16
+ # Those children call .release_inherited first.
17
+ class Evilution::Coverage::LoadTracker
18
+ @paused = false
19
+
20
+ class << self
21
+ # In a forked child that runs tests: stops the counting a tracker paused
22
+ # in the process it was forked from, so the tests find coverage as they
23
+ # would without one. Coverage that anything else set up is left alone.
24
+ def release_inherited
25
+ return unless @paused && ::Coverage.state == :suspended
26
+
27
+ ::Coverage.result(stop: true, clear: true)
28
+ @paused = false
29
+ end
30
+
31
+ def paused!
32
+ @paused = true
33
+ end
34
+ end
35
+
36
+ def start
37
+ return unless ::Coverage.state == :idle
38
+
39
+ ::Coverage.setup(lines: true)
40
+ ::Coverage.resume
41
+ @started = true
42
+ end
43
+
44
+ def pause
45
+ return unless @started && ::Coverage.state == :running
46
+
47
+ ::Coverage.suspend
48
+ self.class.paused!
49
+ end
50
+ end
@@ -3,7 +3,9 @@
3
3
  require_relative "../coverage"
4
4
 
5
5
  # Immutable query over a per-example line-coverage index:
6
- # source file -> line -> [example locations ("spec.rb:line")].
6
+ # source file -> line -> [examples that ran it, as absolute RSpec example ids
7
+ # ("/proj/spec/x_spec.rb[1:2]"); maps cached before ids were recorded hold
8
+ # "/proj/spec/x_spec.rb:line"].
7
9
  # built_files records the source files for which the build completed, so
8
10
  # callers can distinguish "line genuinely uncovered" (file built, no entry)
9
11
  # from "we never built this file" (must fall back, not assert a gap).
@@ -12,8 +14,9 @@ class Evilution::Coverage::Map
12
14
  index = (hash["index"] || {}).transform_values do |lines|
13
15
  lines.transform_keys(&:to_i)
14
16
  end
15
- executed = (hash["executed_lines"] || {}).transform_values { |lines| lines.map(&:to_i) }
16
- new(index: index, built_files: hash["built_files"] || [], executed_lines: executed)
17
+ new(index: index, built_files: hash["built_files"] || [], executed_lines: hash["executed_lines"] || {},
18
+ missed_lines: hash["missed_lines"] || {}, spec_files: hash["spec_files"].to_a,
19
+ loaded_files: hash["loaded_files"].to_a)
17
20
  end
18
21
 
19
22
  # executed_lines records, per file, the lines that ran at all during the build
@@ -21,13 +24,26 @@ class Evilution::Coverage::Map
21
24
  # attributed to no single example). It lets a caller tell a TRUE coverage gap
22
25
  # (line never executed) from a load-covered line an example may still exercise
23
26
  # indirectly -- so the latter falls back instead of being mis-skipped.
24
- def initialize(index:, built_files:, executed_lines: {})
27
+ #
28
+ # missed_lines records, per file, the executable lines that never ran at all,
29
+ # neither while the files loaded nor in any example. A file is listed only
30
+ # when the build vouches for it: every example of the spec files ran, and the
31
+ # file's load was counted. spec_files are the spec files the build ran.
32
+ #
33
+ # loaded_files are the project files the build loaded: a change to any of
34
+ # them can change which lines run, so a cached map is keyed on them too.
35
+ def initialize(index:, built_files:, executed_lines: {}, missed_lines: {}, spec_files: [], loaded_files: [])
25
36
  @index = deep_freeze_index(index)
26
37
  @built_files = built_files.to_a.freeze
27
38
  @executed_lines = deep_freeze_executed(executed_lines)
39
+ @missed_lines = deep_freeze_executed(missed_lines)
40
+ @spec_files = spec_files.to_a.freeze
41
+ @loaded_files = loaded_files.to_a.freeze
28
42
  freeze
29
43
  end
30
44
 
45
+ attr_reader :loaded_files
46
+
31
47
  def examples_for(file, line)
32
48
  @index.dig(file, line) || []
33
49
  end
@@ -41,8 +57,29 @@ class Evilution::Coverage::Map
41
57
  !lines.nil? && lines.include?(line)
42
58
  end
43
59
 
60
+ # Whether no example and no file load executed any of these lines: at least
61
+ # one of them is an executable line that never ran, and none of them ran. A
62
+ # line that is not executable (an `end`, a continuation) says nothing either
63
+ # way, so lines made only of those are not uncovered.
64
+ def uncovered?(file, lines)
65
+ missed = @missed_lines[file]
66
+ return false if missed.nil?
67
+ return false if lines.any? { |line| executed?(file, line) }
68
+
69
+ lines.any? { |line| missed.include?(line) }
70
+ end
71
+
72
+ # Whether the build ran every one of these spec files, so what it saw is
73
+ # what a run of them would see.
74
+ def built_from?(spec_files)
75
+ !spec_files.empty? && spec_files.all? { |file| @spec_files.include?(file) }
76
+ end
77
+
44
78
  def to_h
45
- { "index" => @index, "built_files" => @built_files, "executed_lines" => @executed_lines }
79
+ {
80
+ "index" => @index, "built_files" => @built_files, "executed_lines" => @executed_lines,
81
+ "missed_lines" => @missed_lines, "spec_files" => @spec_files, "loaded_files" => @loaded_files
82
+ }
46
83
  end
47
84
 
48
85
  private
@@ -5,6 +5,7 @@ require "stringio"
5
5
  require_relative "../coverage"
6
6
  require_relative "map"
7
7
  require_relative "recorder"
8
+ require_relative "../integration/rspec/framework_loader"
8
9
 
9
10
  # Builds a per-example coverage Map by running the given spec files under
10
11
  # ::Coverage (lines mode) with the Recorder installed as an around(:each)
@@ -12,16 +13,12 @@ require_relative "recorder"
12
13
  # hook, ::RSpec.world mutation, and ::Coverage state never leak into the
13
14
  # calling process; the parent receives only the serialized map over a pipe.
14
15
  class Evilution::Coverage::MapBuilder
15
- LOCATION_LINE_SUFFIX = /\A(.+?)(:\d+(?::\d+)*)\z/
16
-
17
- # RSpec reports example locations relative to its run dir as "./spec/x.rb:5".
18
- # Store them ABSOLUTE (path expanded against the project root, line suffix
19
- # preserved) so they replay regardless of the per-mutation run's CWD --
20
- # Integration::RSpec#resolve_target passes absolute targets through unchanged.
21
- def self.absolute_location(raw, root)
22
- match = LOCATION_LINE_SUFFIX.match(raw)
23
- path, suffix = match ? [match[1], match[2]] : [raw, ""]
24
- "#{File.expand_path(path, root)}#{suffix}"
16
+ # RSpec reports example ids relative to its run dir ("./spec/x_spec.rb[1:2]").
17
+ # Store them ABSOLUTE so they replay regardless of the per-mutation run's CWD
18
+ # -- Integration::RSpec::TargetResolver passes absolute targets through
19
+ # unchanged. File.expand_path leaves the trailing "[1:2]" as it is.
20
+ def self.absolute_location(id, root)
21
+ File.expand_path(id, root)
25
22
  end
26
23
 
27
24
  def initialize(spec_files:, target_files:, project_root: Evilution::PROJECT_ROOT)
@@ -31,50 +28,154 @@ class Evilution::Coverage::MapBuilder
31
28
  end
32
29
 
33
30
  def call
31
+ @prior_counts = counted_so_far
32
+ Evilution::Coverage::Map.from_h(build_in_fork)
33
+ end
34
+
35
+ private
36
+
37
+ def build_in_fork
34
38
  read_io, write_io = IO.pipe
35
39
  read_io.binmode
36
40
  write_io.binmode
37
41
  pid = fork do
38
42
  read_io.close
39
- Marshal.dump(build_in_child.to_h, write_io)
43
+ Marshal.dump(child_payload, write_io)
40
44
  write_io.close
41
45
  exit!(0)
42
46
  end
43
47
  write_io.close
44
48
  payload = read_io.read
45
49
  read_io.close
46
- Process.wait(pid)
47
- # Trust boundary: payload is a Marshal dump this process's own forked child
48
- # produced over a private pipe -- same rationale as Channel::Frame.
49
- Evilution::Coverage::Map.from_h(Marshal.load(payload))
50
+ _, status = Process.wait2(pid)
51
+ unpack(payload, status)
50
52
  end
51
53
 
52
- private
54
+ # A child that fails sends its error back instead of dying with a backtrace:
55
+ # the caller falls back to lexical targeting and says why in one line, so
56
+ # only the first line of a message that runs to more (a SyntaxError's) goes.
57
+ #
58
+ # Nothing need have loaded RSpec in this process: under fork isolation the
59
+ # canary runs it in a child of its own, and without --preload neither does
60
+ # the project. Loading it here leaves the parent without it, as before.
61
+ def child_payload
62
+ Evilution::Integration::RSpec::FrameworkLoader.new.call
63
+ { map: build_in_child.to_h }
64
+ rescue StandardError, ScriptError => e
65
+ { error: "#{e.class}: #{e.message.lines.first.to_s.chomp}" }
66
+ end
67
+
68
+ def unpack(payload, status)
69
+ result = decode(payload)
70
+ raise Evilution::Error, "coverage map builder exited without a map (#{status})" unless result
71
+ raise Evilution::Error, result[:error] if result[:error]
72
+
73
+ result[:map]
74
+ end
75
+
76
+ # Nothing, or a dump cut short by a child killed while writing it, is no map.
77
+ # Trust boundary: payload is a Marshal dump this process's own forked child
78
+ # produced over a private pipe -- same rationale as Channel::Frame.
79
+ def decode(payload)
80
+ Marshal.load(payload)
81
+ rescue ArgumentError, TypeError
82
+ nil
83
+ end
53
84
 
54
85
  # Runs entirely inside the forked child.
55
86
  def build_in_child
56
- recorder = Evilution::Coverage::Recorder.new(target_files: @target_files)
57
- ::Coverage.start(lines: true) unless ::Coverage.running?
87
+ recorder = Evilution::Coverage::Recorder.new(target_files: @target_files, prior_counts: @prior_counts)
88
+ unvouched = loaded_uncounted
89
+ count_lines
58
90
  # The child inherited the parent's fully-loaded RSpec.world; wipe it so the
59
91
  # nested runner executes ONLY @spec_files and never re-enters the host suite
60
92
  # (which would recursively fork this builder).
61
93
  reset_world
62
94
  install_hook(recorder)
95
+ forks = watch_forks
63
96
  ::RSpec::Core::Runner.run(@spec_files, StringIO.new, StringIO.new)
64
- recorder.to_map(built_files: @target_files)
97
+ vouched = run_complete? && !forks[:forked] ? @target_files - unvouched : []
98
+ recorder.to_map(built_files: @target_files, vouched_files: vouched, spec_files: @spec_files,
99
+ loaded_files: loaded_project_files)
100
+ end
101
+
102
+ # The files of the project the build loaded: required, or compiled while
103
+ # lines were counted (RSpec loads a spec file without requiring it). Files of
104
+ # installed gems change only with the lockfile, which the cache is keyed on.
105
+ def loaded_project_files
106
+ roots = [@project_root, File.realpath(@project_root)].uniq.map { |root| "#{root}/" }
107
+ gems = ::Gem.path.map { |dir| "#{dir}/" }
108
+ files = ($LOADED_FEATURES + ::Coverage.peek_result.keys).uniq
109
+ files.select { |file| file.start_with?(*roots) && !file.start_with?(*gems) && File.file?(file) }
110
+ end
111
+
112
+ # The parent may have started counting before its preload and paused it
113
+ # since (Evilution::Coverage::LoadTracker). Read in the parent: a forked
114
+ # child starts its counts from zero.
115
+ def counted_so_far
116
+ ::Coverage.state == :idle ? {} : ::Coverage.peek_result
117
+ end
118
+
119
+ def count_lines
120
+ case ::Coverage.state
121
+ when :suspended then ::Coverage.resume
122
+ when :idle then ::Coverage.start(lines: true)
123
+ end
124
+ end
125
+
126
+ # Target files already loaded with every line at zero: they loaded while
127
+ # nothing was counting, so the lines that ran as they loaded (a class body,
128
+ # a `def`, a constant) read as never run. No line of theirs can be vouched for.
129
+ def loaded_uncounted
130
+ @target_files.select do |file|
131
+ lines = line_counts(@prior_counts[file])
132
+ lines && lines.none? { |count| count.is_a?(Integer) && count.positive? }
133
+ end
134
+ end
135
+
136
+ def line_counts(entry)
137
+ entry.is_a?(Hash) ? entry[:lines] : entry
138
+ end
139
+
140
+ # Whether an example forked: a forked process counts its lines in its own
141
+ # copy of the counters, which this build never reads, so what only it ran
142
+ # would read as never run -- while a mutation applied before the fork
143
+ # reaches it, and its tests can kill the mutation there. Kernel#fork,
144
+ # Process.fork and IO.popen("-") all go through Process._fork. Installed in
145
+ # the build's own child, which exits once the map is sent.
146
+ def watch_forks
147
+ forks = { forked: false }
148
+ hook = Module.new do
149
+ define_method(:_fork) do
150
+ forks[:forked] = true
151
+ super()
152
+ end
153
+ end
154
+ ::Process.singleton_class.prepend(hook)
155
+ forks
156
+ end
157
+
158
+ # Whether every example of the spec files ran: a spec file that failed to
159
+ # load runs none, a run that stopped early (fail-fast, an interrupt) fewer
160
+ # than it loaded, and either leaves lines at zero that a full run executes.
161
+ def run_complete?
162
+ loaded = ::RSpec.world.example_count
163
+ loaded.positive? && ::RSpec.configuration.reporter.examples.length == loaded
65
164
  end
66
165
 
67
166
  def reset_world
68
167
  ::RSpec.respond_to?(:clear_examples) ? ::RSpec.clear_examples : ::RSpec.reset
69
168
  end
70
169
 
170
+ # Each example is recorded by its id ("./spec/x_spec.rb[1:2:1]"), not its
171
+ # :location. An example from a shared group has its :location in the shared
172
+ # file, which on its own defines no example (and may not even load); its id
173
+ # names the spec file that includes the group and picks out that one example.
71
174
  def install_hook(recorder)
72
175
  root = @project_root
73
176
  ::RSpec.configure do |config|
74
177
  config.around(:each) do |example|
75
- location = Evilution::Coverage::MapBuilder.absolute_location(
76
- example.metadata[:location].to_s, root
77
- )
178
+ location = Evilution::Coverage::MapBuilder.absolute_location(example.id, root)
78
179
  recorder.around_example(location) { example.run }
79
180
  end
80
181
  end