evilution 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +1 -0
  3. data/.rubocop_todo.yml +30 -0
  4. data/CHANGELOG.md +59 -0
  5. data/README.md +285 -45
  6. data/docs/architecture.md +110 -12
  7. data/docs/ast_pattern_syntax.md +35 -7
  8. data/docs/combined_runs_spike.md +407 -0
  9. data/docs/integrations.md +1 -1
  10. data/docs/isolation.md +63 -2
  11. data/docs/migration-from-mutant.md +16 -11
  12. data/docs/mutant_schemata_spike.md +387 -0
  13. data/docs/operator_yield_study.md +576 -0
  14. data/docs/probe_first_spike.md +264 -0
  15. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  16. data/lib/evilution/ast/callback_declaration.rb +36 -3
  17. data/lib/evilution/ast/class_body.rb +62 -16
  18. data/lib/evilution/ast/def_region.rb +130 -0
  19. data/lib/evilution/ast/local_declaration.rb +37 -0
  20. data/lib/evilution/ast/parent_index.rb +50 -0
  21. data/lib/evilution/ast/parse_cache.rb +27 -0
  22. data/lib/evilution/ast/parser.rb +8 -3
  23. data/lib/evilution/ast/pattern/filter.rb +53 -12
  24. data/lib/evilution/ast/skip_filter.rb +83 -0
  25. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  26. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  27. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  28. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  29. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  30. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  31. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  32. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  33. data/lib/evilution/ast/skip_sites.rb +9 -0
  34. data/lib/evilution/ast/source_surgeon.rb +20 -2
  35. data/lib/evilution/ast/syntax_check.rb +10 -8
  36. data/lib/evilution/baseline/report.rb +11 -2
  37. data/lib/evilution/baseline.rb +31 -6
  38. data/lib/evilution/cache/keys.rb +140 -0
  39. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  40. data/lib/evilution/cache/skeleton.rb +57 -0
  41. data/lib/evilution/cache/spec_digest.rb +39 -0
  42. data/lib/evilution/cache.rb +145 -42
  43. data/lib/evilution/class_tree.rb +38 -0
  44. data/lib/evilution/cli/command.rb +12 -4
  45. data/lib/evilution/cli/commands/run.rb +12 -3
  46. data/lib/evilution/cli/commands/subjects.rb +5 -2
  47. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  48. data/lib/evilution/cli/printers/environment.rb +34 -4
  49. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  50. data/lib/evilution/compare/categorizer.rb +2 -1
  51. data/lib/evilution/compare/normalizer.rb +3 -2
  52. data/lib/evilution/config/sources.rb +3 -0
  53. data/lib/evilution/config/validators/base.rb +9 -0
  54. data/lib/evilution/config/validators/since.rb +22 -0
  55. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  56. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  58. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  59. data/lib/evilution/config.rb +122 -6
  60. data/lib/evilution/coverage/load_tracker.rb +50 -0
  61. data/lib/evilution/coverage/map.rb +42 -5
  62. data/lib/evilution/coverage/map_builder.rb +123 -22
  63. data/lib/evilution/coverage/map_store.rb +15 -5
  64. data/lib/evilution/coverage/recorder.rb +42 -3
  65. data/lib/evilution/coverage_example_filter.rb +35 -0
  66. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  67. data/lib/evilution/example_filter.rb +20 -0
  68. data/lib/evilution/git/changed_lines.rb +103 -0
  69. data/lib/evilution/git/since_scope.rb +77 -0
  70. data/lib/evilution/git/unified_diff.rb +68 -0
  71. data/lib/evilution/infra_crash.rb +26 -0
  72. data/lib/evilution/integration/base.rb +97 -6
  73. data/lib/evilution/integration/crash_detector.rb +46 -6
  74. data/lib/evilution/integration/known_failures.rb +4 -0
  75. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  76. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  77. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  78. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  79. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  80. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  81. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  82. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  83. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  84. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  85. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  86. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  87. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  88. data/lib/evilution/integration/minitest.rb +22 -2
  89. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  90. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  91. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  92. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  93. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  94. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  95. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  96. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  97. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  98. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  99. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  100. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  101. data/lib/evilution/integration/rspec.rb +98 -47
  102. data/lib/evilution/integration/stop_policy.rb +49 -0
  103. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  104. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  105. data/lib/evilution/integration/test_unit.rb +28 -2
  106. data/lib/evilution/isolation/fork.rb +58 -13
  107. data/lib/evilution/isolation/in_process.rb +12 -1
  108. data/lib/evilution/line_set.rb +72 -0
  109. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  110. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  111. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  112. data/lib/evilution/mcp/info_tool.rb +24 -17
  113. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  114. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  115. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  116. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  117. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  118. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  119. data/lib/evilution/mcp/session_tool.rb +1 -2
  120. data/lib/evilution/mcp.rb +4 -0
  121. data/lib/evilution/mutation/line_diff.rb +111 -0
  122. data/lib/evilution/mutation.rb +17 -13
  123. data/lib/evilution/mutator/base.rb +121 -98
  124. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  125. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  126. data/lib/evilution/mutator/load_sources.rb +88 -0
  127. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  128. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  129. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  130. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  131. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  132. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  133. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  135. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  136. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  137. data/lib/evilution/mutator/registry.rb +11 -5
  138. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  139. data/lib/evilution/parallel/pool.rb +11 -4
  140. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  141. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  142. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  143. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  144. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  145. data/lib/evilution/parallel/work_queue.rb +69 -12
  146. data/lib/evilution/parallel_db_warning.rb +39 -12
  147. data/lib/evilution/process_supervisor.rb +22 -2
  148. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  149. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  150. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  151. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  152. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  153. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  154. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  155. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  156. data/lib/evilution/reporter/cli.rb +9 -2
  157. data/lib/evilution/reporter/html/assets/style.css +8 -0
  158. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  159. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  160. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  161. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  162. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  163. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  164. data/lib/evilution/reporter/json/subjects.rb +1 -0
  165. data/lib/evilution/reporter/json/timings.rb +42 -0
  166. data/lib/evilution/reporter/json.rb +13 -50
  167. data/lib/evilution/result/coverage_gap.rb +5 -0
  168. data/lib/evilution/result/mutation_result.rb +55 -3
  169. data/lib/evilution/result/mutation_timings.rb +54 -0
  170. data/lib/evilution/result/run_timings.rb +20 -0
  171. data/lib/evilution/result/subject_score.rb +8 -1
  172. data/lib/evilution/result/subject_scorer.rb +5 -3
  173. data/lib/evilution/result/summary.rb +49 -12
  174. data/lib/evilution/runner/baseline_runner.rb +47 -5
  175. data/lib/evilution/runner/diagnostics.rb +0 -13
  176. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  177. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  178. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  179. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  180. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  181. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  182. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  183. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  184. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  185. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  186. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  187. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  190. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  191. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  195. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  196. data/lib/evilution/runner/mutation_executor.rb +75 -10
  197. data/lib/evilution/runner/mutation_planner.rb +29 -21
  198. data/lib/evilution/runner/phase_clock.rb +22 -0
  199. data/lib/evilution/runner/report_publisher.rb +1 -1
  200. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  201. data/lib/evilution/runner.rb +87 -26
  202. data/lib/evilution/session/diff.rb +9 -2
  203. data/lib/evilution/session/store.rb +17 -2
  204. data/lib/evilution/source_ast_cache.rb +10 -0
  205. data/lib/evilution/spec_ast_cache.rb +15 -0
  206. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  207. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  208. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  209. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  210. data/lib/evilution/spec_resolver.rb +76 -65
  211. data/lib/evilution/spec_selector.rb +17 -13
  212. data/lib/evilution/subject.rb +7 -2
  213. data/lib/evilution/version.rb +1 -1
  214. data/lib/evilution/worker_database/copies.rb +73 -0
  215. data/lib/evilution/worker_database/schema.rb +39 -0
  216. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  217. data/lib/evilution/worker_database.rb +214 -0
  218. data/schema/evilution.config.schema.json +74 -2
  219. metadata +84 -13
  220. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  221. data/scripts/benchmark_density +0 -262
  222. data/scripts/benchmark_density.yml +0 -19
  223. data/scripts/canary_manifest.yml +0 -47
  224. data/scripts/compare_mutations +0 -421
  225. data/scripts/compare_mutations.yml +0 -24
  226. data/scripts/compare_targeting +0 -279
  227. data/scripts/compare_targeting.example.yml +0 -24
  228. data/scripts/mutant_json_adapter +0 -227
@@ -1,9 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "../integration"
4
+ require_relative "stop_policy"
4
5
 
5
6
  class Evilution::Integration::MinitestCrashDetector
7
+ # The run's stop policy. Left alone, the detector never stops a run: only a
8
+ # mutation's own run is handed a policy that can.
9
+ attr_writer :policy
10
+
6
11
  def initialize
12
+ @policy = Evilution::Integration::StopPolicy::DISABLED
7
13
  reset
8
14
  end
9
15
 
@@ -28,12 +34,17 @@ class Evilution::Integration::MinitestCrashDetector
28
34
  @crashes = []
29
35
  end
30
36
 
37
+ # UnexpectedError and Skip both subclass Assertion. A crash hands its
38
+ # exception to the policy and an assertion failure nothing; a skip says
39
+ # nothing about the mutant, so the policy never hears of it.
31
40
  def record(result)
32
41
  result.failures.each do |failure|
33
42
  if failure.is_a?(::Minitest::UnexpectedError)
34
43
  @crashes << failure.error
44
+ @policy.record(id: test_id(result), exception: failure.error)
35
45
  elsif failure.is_a?(::Minitest::Assertion)
36
46
  @assertion_failures += 1
47
+ @policy.record(id: test_id(result)) unless failure.is_a?(::Minitest::Skip)
37
48
  end
38
49
  end
39
50
  end
@@ -59,4 +70,12 @@ class Evilution::Integration::MinitestCrashDetector
59
70
  def unique_crash_classes
60
71
  @crashes.map { |e| e.class.name }.uniq
61
72
  end
73
+
74
+ private
75
+
76
+ # TestIds is loaded by the Minitest integration, the detector's only user.
77
+ # Requiring it here would load that integration from this file, mid-load.
78
+ def test_id(result)
79
+ Evilution::Integration::Minitest::TestIds.of(result)
80
+ end
62
81
  end
@@ -33,12 +33,13 @@ class Evilution::Integration::RSpec::BaselineRunner
33
33
  # What RSpec printed is the only account of a failure outside any example --
34
34
  # a file that does not load, a suite hook that raises.
35
35
  def report(passed, output)
36
- return Evilution::Baseline::Report.build(passed: true) if passed
36
+ durations = Evilution::Integration::RSpec::ExampleIds.durations(::RSpec.world)
37
+ return Evilution::Baseline::Report.build(passed: true, example_durations: durations) if passed
37
38
 
38
39
  failures = failed_examples
39
40
  Evilution::Baseline::Report.build(
40
41
  passed: false, failures: failures, error: failures.empty? ? output : nil,
41
- failed_ids: Evilution::Integration::RSpec::ExampleIds.failed(::RSpec.world)
42
+ failed_ids: Evilution::Integration::RSpec::ExampleIds.failed(::RSpec.world), example_durations: durations
42
43
  )
43
44
  end
44
45
 
@@ -17,8 +17,13 @@ class Evilution::Integration::RSpec::CrashDetectorLifecycle
17
17
  # The run rebuilds RSpec's formatter loader so its formatters are bound to the
18
18
  # run's own streams (EV-m6xc / GH #1627), which drops any formatter registered
19
19
  # before it — the detector included. It is registered per run instead.
20
- def register
20
+ #
21
+ # The policy is reset with it: one latched by the previous run would stop
22
+ # this one at its first failure, decisive or not.
23
+ def register(policy)
24
+ policy.reset
21
25
  detector = current
26
+ detector.policy = policy
22
27
  ::RSpec.configuration.add_formatter(detector)
23
28
  detector
24
29
  end
@@ -7,6 +7,12 @@ module Evilution::Integration::RSpec::ExampleFilterApplier
7
7
  def call(_mutation, files)
8
8
  files
9
9
  end
10
+
11
+ def warm(_mutation, _files); end
12
+
13
+ def uncovered?(_mutation, _files)
14
+ false
15
+ end
10
16
  end
11
17
 
12
18
  class Custom
@@ -17,5 +23,15 @@ module Evilution::Integration::RSpec::ExampleFilterApplier
17
23
  def call(mutation, files)
18
24
  @filter.call(mutation, files)
19
25
  end
26
+
27
+ # Only a filter that caches what it parses has anything to warm.
28
+ def warm(mutation, files)
29
+ @filter.warm(mutation, files) if @filter.respond_to?(:warm)
30
+ end
31
+
32
+ # Only a filter backed by line coverage knows which lines no example runs.
33
+ def uncovered?(mutation, files)
34
+ @filter.respond_to?(:uncovered?) && @filter.uncovered?(mutation, files)
35
+ end
20
36
  end
21
37
  end
@@ -11,6 +11,11 @@ require_relative "../rspec"
11
11
  # resolved before the position within the file is added.
12
12
  module Evilution::Integration::RSpec::ExampleIds
13
13
  FAILED = :failed
14
+ # An example with any of these statuses was run; one RSpec skipped, or never
15
+ # reached because the run stopped, has no status at all.
16
+ EXECUTED = %i[passed failed pending].freeze
17
+ # A pending example's time says nothing about what running it costs.
18
+ TIMED = %i[passed failed].freeze
14
19
 
15
20
  def self.of(example)
16
21
  metadata = example.metadata
@@ -26,6 +31,24 @@ module Evilution::Integration::RSpec::ExampleIds
26
31
  .map { |example| of(example) }
27
32
  end
28
33
 
34
+ # How long each example that ran took, in seconds, by id.
35
+ def self.durations(world)
36
+ return {} unless world.respond_to?(:all_examples)
37
+
38
+ world.all_examples.each_with_object({}) do |example, durations|
39
+ result = example.execution_result
40
+ durations[of(example)] = result.run_time if TIMED.include?(result.status)
41
+ end
42
+ end
43
+
44
+ # How many examples the run in RSpec.world executed; nil when the world
45
+ # cannot list them.
46
+ def self.executed_count(world)
47
+ return nil unless world.respond_to?(:all_examples)
48
+
49
+ world.all_examples.count { |example| EXECUTED.include?(example.execution_result.status) }
50
+ end
51
+
29
52
  def self.canonical(path)
30
53
  File.realpath(path)
31
54
  rescue SystemCallError
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../rspec"
4
+ require_relative "example_ids"
5
+
6
+ # The order a mutation's run tries its examples in, when it stops at the
7
+ # first kill: the cost of a kill is whatever runs before it.
8
+ #
9
+ # First the examples that killed this mutation's neighbours earlier in the
10
+ # run, in the order they were given; then the rest, cheapest first by the time
11
+ # each took in the baseline. Examples without a time come last, and examples
12
+ # that compare equal keep the order they were defined in.
13
+ #
14
+ # RSpec asks an ordering for the examples of one group and, separately, for
15
+ # its child groups, so examples never leave their group and a group's
16
+ # `before(:context)` hooks still run once, around all of its examples. A group
17
+ # goes first when any example under it is a killer, otherwise by the time its
18
+ # timed examples take in all.
19
+ #
20
+ # Registered with RSpec under NAME and chosen with `--order NAME`. Outside a
21
+ # run (between #start and #finish) it keeps the order it is given, so a
22
+ # process whose global ordering it is left as runs everything as defined.
23
+ class Evilution::Integration::RSpec::ExampleOrder
24
+ NAME = :evilution_killer_first
25
+ UNKNOWN = Float::INFINITY
26
+ private_constant :UNKNOWN
27
+
28
+ # Whether RSpec runs an ordering named on the command line. An older one
29
+ # ignores a name it does not know and keeps whatever order it had.
30
+ def self.supported?
31
+ defined?(::RSpec::Core::Ordering::Delayed) ? true : false
32
+ end
33
+
34
+ # durations: seconds per example id (see ExampleIds) from the baseline.
35
+ def initialize(durations:)
36
+ @durations = durations
37
+ finish
38
+ end
39
+
40
+ # Whether there is anything to order by: without killers or durations every
41
+ # example compares equal and the defined order stands.
42
+ def informed?(killers)
43
+ !(@durations.empty? && killers.empty?)
44
+ end
45
+
46
+ # killers: the ids of the examples to run first, the most telling first.
47
+ def start(killers)
48
+ @ranks = {}
49
+ killers.each_with_index { |id, rank| @ranks[id] ||= rank }
50
+ @ids = {}.compare_by_identity
51
+ @active = true
52
+ end
53
+
54
+ def finish
55
+ @active = false
56
+ @ranks = {}
57
+ @ids = {}.compare_by_identity
58
+ end
59
+
60
+ def order(items)
61
+ return items unless @active
62
+
63
+ items.each_with_index.sort_by { |item, index| [*key(item), index] }.map(&:first)
64
+ end
65
+
66
+ private
67
+
68
+ def key(item)
69
+ item.respond_to?(:descendant_filtered_examples) ? group_key(item) : example_key(item)
70
+ end
71
+
72
+ def example_key(example)
73
+ id = (@ids[example] ||= Evilution::Integration::RSpec::ExampleIds.of(example))
74
+ [@ranks.fetch(id, UNKNOWN), @durations.fetch(id, UNKNOWN)]
75
+ end
76
+
77
+ # A group's cost is what its timed examples take: an example without a time
78
+ # (pending, or past what a report keeps per file) says nothing of the others.
79
+ # A group with examples but no time for any of them comes after the timed.
80
+ def group_key(group)
81
+ keys = group.descendant_filtered_examples.map { |example| example_key(example) }
82
+ known = keys.map(&:last).reject { |seconds| seconds == UNKNOWN }
83
+ cost = known.empty? && !keys.empty? ? UNKNOWN : known.sum
84
+ [keys.map(&:first).min || UNKNOWN, cost]
85
+ end
86
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../rspec"
4
+ require_relative "example_order"
5
+
6
+ # Whether a mutation's run asks RSpec for the killer-first order (see
7
+ # ExampleOrder), and that order put in place for the run.
8
+ #
9
+ # Only when enabled: the user asked for it, and the run stops at its first
10
+ # kill (one that goes through every example costs the same in any order). A
11
+ # suite whose examples lean on the ones defined before them fails in another
12
+ # order whatever the mutant, and would report kills nobody can reproduce, so
13
+ # the defined order is the default. Without killers or baseline times there is
14
+ # nothing to order by, and the defined order stands as well.
15
+ class Evilution::Integration::RSpec::KillerFirst
16
+ def initialize(enabled:, durations:)
17
+ @enabled = enabled
18
+ @order = Evilution::Integration::RSpec::ExampleOrder.new(durations: durations)
19
+ @killers = []
20
+ @suspended = false
21
+ end
22
+
23
+ # The ids of the examples to run first, the most telling first; kept until
24
+ # the next call.
25
+ def prioritize(test_ids)
26
+ @killers = Array(test_ids)
27
+ nil
28
+ end
29
+
30
+ def applies?
31
+ reorders? && Evilution::Integration::RSpec::ExampleOrder.supported?
32
+ end
33
+
34
+ # Whether the next run is meant to go in the killer-first order, as far as a
35
+ # process that has not loaded RSpec can tell: the parent asks, to know
36
+ # whether a run that timed out may owe that to the order.
37
+ def reorders?
38
+ @enabled && !@suspended && @order.informed?(@killers)
39
+ end
40
+
41
+ # Runs the block with every run in it left in the defined order.
42
+ def suspended
43
+ @suspended = true
44
+ yield
45
+ ensure
46
+ @suspended = false
47
+ end
48
+
49
+ # RSpec applies `--order` only to the first run in a process that names
50
+ # one: from then on the order counts as forced, and naming another changes
51
+ # nothing. A process that has already run RSpec -- the canary, a run before
52
+ # this one -- would keep the defined order. So the ordering is made the
53
+ # global one directly, as well as registered under the name the run's
54
+ # arguments give, which is what decides when this run is the first.
55
+ #
56
+ # Afterwards the global ordering is the defined one, as the `--order
57
+ # defined` of every other evilution run leaves it.
58
+ def during(ordered)
59
+ return yield unless ordered
60
+
61
+ registry = ::RSpec.configuration.ordering_registry
62
+ registry.register(Evilution::Integration::RSpec::ExampleOrder::NAME, @order)
63
+ registry.register(:global, @order)
64
+ @order.start(@killers)
65
+ yield
66
+ ensure
67
+ if ordered
68
+ @order.finish
69
+ registry.register(:global, registry.fetch(:defined))
70
+ end
71
+ end
72
+ end
@@ -61,4 +61,11 @@ class Evilution::Integration::RSpec::ResultBuilder
61
61
 
62
62
  { passed: false, test_command: command }
63
63
  end
64
+
65
+ # killing_example: the id of the example whose failure killed, nil when
66
+ # none did. examples_run: how many examples the run executed, nil when
67
+ # RSpec could not say. Either is left out when nil.
68
+ def with_run_facts(result, killing_example:, examples_run:)
69
+ result.merge({ killing_example: killing_example, examples_run: examples_run }.compact)
70
+ end
64
71
  end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../state_guard"
4
+ require_relative "../../rspec"
5
+ require_relative "../../../class_tree"
6
+
7
+ # Empties the example groups a run defined of their constants and instance
8
+ # variables, so that what they hold does not outlive the run in a process
9
+ # that runs more than one.
10
+ #
11
+ # The groups are found below ExampleGroup in the class tree rather than among
12
+ # every class on the heap: a heap walk, twice a run, touched every object the
13
+ # process holds.
14
+ class Evilution::Integration::RSpec::StateGuard::ExampleGroupDescendants
15
+ def snapshot
16
+ Set.new(descendants.map(&:object_id))
17
+ end
18
+
19
+ def release(eg_before)
20
+ return unless eg_before
21
+
22
+ descendants.each do |klass|
23
+ next if eg_before.include?(klass.object_id)
24
+
25
+ klass.constants(false).each do |const|
26
+ klass.send(:remove_const, const)
27
+ rescue NameError
28
+ # Constant may have been removed concurrently (e.g. via autoload
29
+ # reload) between #constants(false) and #remove_const. Best-effort
30
+ # cleanup — nothing to do if it's already gone.
31
+ end
32
+
33
+ klass.instance_variables.each do |ivar|
34
+ klass.remove_instance_variable(ivar)
35
+ end
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def descendants
42
+ Evilution::ClassTree.descendants(::RSpec::Core::ExampleGroup)
43
+ end
44
+ end
@@ -7,7 +7,7 @@ require_relative "../rspec"
7
7
  # children loaded, so the namespace must exist first.
8
8
  class Evilution::Integration::RSpec::StateGuard; end # rubocop:disable Lint/EmptyClass
9
9
 
10
- require_relative "state_guard/object_space_example_groups"
10
+ require_relative "state_guard/example_group_descendants"
11
11
  require_relative "state_guard/world_example_groups"
12
12
  require_relative "state_guard/world_sources_by_path"
13
13
  require_relative "state_guard/world_filtered_examples"
@@ -19,7 +19,7 @@ require_relative "../../diagnostic"
19
19
 
20
20
  class Evilution::Integration::RSpec::StateGuard
21
21
  DEFAULT_STRATEGIES = [
22
- ObjectSpaceExampleGroups.new,
22
+ ExampleGroupDescendants.new,
23
23
  WorldExampleGroups.new,
24
24
  WorldSourcesByPath.new,
25
25
  WorldFilteredExamples.new,
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../rspec"
4
+
5
+ # Targets are passed straight through when they resolve against Dir.pwd,
6
+ # preserving the relative-path contract for callers/tests. When the CWD
7
+ # cannot find them — workers chdir'd into a per-mutation sandbox by the
8
+ # isolator — the targets are expanded against
9
+ # Evilution::PROJECT_ROOT so RSpec::Core::Runner can still load the files.
10
+ class Evilution::Integration::RSpec::TargetResolver
11
+ # Splits a target into (path, line). RSpec example locations end in
12
+ # `:LINE` (or `:LINE:LINE...` for nested groups), so the suffix is
13
+ # anchored to the END of the string — that way path components which
14
+ # themselves contain colons (e.g. a Windows-style `C:/proj/spec/x.rb`)
15
+ # are not mis-split by a naive `partition(":")`.
16
+ TARGET_LINE_SUFFIX = /\A(.+?)(:\d+(?::\d+)*)\z/
17
+ private_constant :TARGET_LINE_SUFFIX
18
+
19
+ def call(targets)
20
+ targets.map { |target| resolve(target) }
21
+ end
22
+
23
+ def resolve(target)
24
+ return target if target.start_with?("/")
25
+
26
+ if (match = TARGET_LINE_SUFFIX.match(target))
27
+ path = match[1]
28
+ line_suffix = match[2]
29
+ else
30
+ path = target
31
+ line_suffix = ""
32
+ end
33
+
34
+ return target if File.exist?(path)
35
+ return target unless Evilution.in_isolated_worker?
36
+
37
+ absolute = File.expand_path(path, Evilution::PROJECT_ROOT)
38
+ return target unless File.exist?(absolute)
39
+
40
+ "#{absolute}#{line_suffix}"
41
+ end
42
+ end
@@ -13,12 +13,14 @@ class Evilution::Integration::RSpec::TestFileResolver
13
13
  @warner = warner
14
14
  end
15
15
 
16
- def call(mutation)
16
+ # warn: false resolves without telling the user about a file nothing
17
+ # resolves for, as when the files are only looked up ahead of the run.
18
+ def call(mutation, warn: true)
17
19
  return @test_files if @test_files
18
20
 
19
21
  resolved = Array(@spec_selector.call(mutation.file_path))
20
22
  if resolved.empty?
21
- @warner.call(mutation.file_path, fallback_to_full_suite: @fallback_to_full_suite)
23
+ @warner.call(mutation.file_path, fallback_to_full_suite: @fallback_to_full_suite) if warn
22
24
  return @fallback_to_full_suite ? ["spec"] : nil
23
25
  end
24
26
 
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "forwardable"
4
+ require "shellwords"
3
5
  require "stringio"
4
6
  require_relative "base"
5
7
  require_relative "known_failures"
8
+ require_relative "stop_policy"
6
9
  require_relative "../spec_resolver"
7
10
  require_relative "../spec_selector"
8
11
  require_relative "../related_spec_heuristic"
@@ -10,6 +13,8 @@ require_relative "../related_spec_heuristic"
10
13
  require_relative "../integration"
11
14
 
12
15
  class Evilution::Integration::RSpec < Evilution::Integration::Base
16
+ extend Forwardable
17
+
13
18
  def self.baseline_runner
14
19
  BaselineRunner.new
15
20
  end
@@ -26,6 +31,9 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
26
31
  spec_selector: nil,
27
32
  example_filter: nil,
28
33
  known_failures: [],
34
+ stop_on_kill: false,
35
+ killer_first: false,
36
+ example_durations: {},
29
37
  framework_loader: FrameworkLoader.new,
30
38
  test_file_resolver: nil,
31
39
  example_filter_applier: nil,
@@ -47,9 +55,42 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
47
55
  @result_builder = result_builder
48
56
  @state_guard = state_guard
49
57
  @known_failures = Evilution::Integration::KnownFailures.new(known_failures)
58
+ @stop_policy = Evilution::Integration::StopPolicy.new(enabled: stop_on_kill, known_failures: @known_failures)
59
+ @killer_first = KillerFirst.new(enabled: stop_on_kill && killer_first, durations: example_durations)
60
+ @target_resolver = TargetResolver.new
50
61
  super(hooks: hooks)
51
62
  end
52
63
 
64
+ # Taken by the runs that follow, until the next call (see KillerFirst).
65
+ def prioritize(test_ids)
66
+ @killer_first.prioritize(test_ids)
67
+ end
68
+
69
+ def_delegator :@killer_first, :reorders?
70
+ def_delegator :@killer_first, :suspended, :in_defined_order
71
+
72
+ # One mutation per source file: every mutation in a file shares its source
73
+ # and the spec files the selector resolves for it. The related-spec
74
+ # heuristic, when on, adds files per mutation from its diff; those are
75
+ # warmed only if the file's first mutation adds them, and are otherwise
76
+ # parsed by the child that runs the mutation, as without warming.
77
+ def warm_targeting(mutations)
78
+ mutations.uniq(&:file_path).each do |mutation|
79
+ files = @test_file_resolver.call(mutation, warn: false)
80
+ @example_filter_applier.warm(mutation, files) unless files.nil?
81
+ end
82
+ end
83
+
84
+ # Whether no example of the spec files this mutation would run executes the
85
+ # lines it changes (see Evilution::CoverageExampleFilter#uncovered?). Asked in
86
+ # the parent, ahead of the run, like #warm_targeting.
87
+ def uncovered?(mutation)
88
+ files = @test_file_resolver.call(mutation, warn: false)
89
+ return false if files.nil?
90
+
91
+ @example_filter_applier.uncovered?(mutation, files)
92
+ end
93
+
53
94
  private
54
95
 
55
96
  def build_example_filter_applier(example_filter)
@@ -73,10 +114,20 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
73
114
  targets = @example_filter_applier.call(mutation, files)
74
115
  return @result_builder.unresolved_example(mutation) if targets.nil?
75
116
 
76
- result = run_targets(targets)
117
+ result = run_targets(targets, ordered: @killer_first.applies?)
77
118
  return result unless confirm?(result, targets, files)
78
119
 
79
- run_targets(files)
120
+ confirm_run(result, files)
121
+ end
122
+
123
+ # A survivor's confirming run is left in the defined order, so its verdict
124
+ # comes from the file as it always ran. The examples both runs ran count.
125
+ def confirm_run(first, files)
126
+ started = monotonic_now
127
+ confirmed = run_targets(files)
128
+ examples_run = [first[:examples_run], confirmed[:examples_run]].compact
129
+ confirmed = confirmed.merge(examples_run: examples_run.sum) unless examples_run.empty?
130
+ confirmed.merge(timings: { confirmation: monotonic_now - started })
80
131
  end
81
132
 
82
133
  # Targeting runs a subset of the examples in a spec file, chosen by name. A
@@ -96,68 +147,65 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
96
147
  (result[:passed] || result[:known_failures_only]) && targets != files
97
148
  end
98
149
 
99
- def run_targets(targets)
100
- args = ["--format", "progress", "--no-color", "--order", "defined", *resolve_targets(targets)]
101
- command = "rspec #{args.join(" ")}"
150
+ # The command reported is the run in the defined order: it is what a reader
151
+ # can run to reproduce the verdict, and the killer-first ordering exists
152
+ # only inside evilution. It is shell-escaped so it pastes as is: an example
153
+ # id ("spec/x_spec.rb[1:2]") is a glob to zsh.
154
+ def run_targets(targets, ordered: false)
155
+ files = @target_resolver.call(targets)
156
+ command = "rspec #{Shellwords.join(run_args("defined", files))}"
157
+ args = ordered ? run_args(ExampleOrder::NAME, files) : run_args("defined", files)
102
158
 
103
159
  reset_examples
104
- execute_run(args, command)
105
- end
106
-
107
- # Targets are passed straight through when they resolve against Dir.pwd,
108
- # preserving the relative-path contract for callers/tests. When the CWD
109
- # cannot find them — workers chdir'd into a per-mutation sandbox by the
110
- # isolator (EV-wqxu / GH #1278) — the targets are expanded against
111
- # Evilution::PROJECT_ROOT so RSpec::Core::Runner can still load the files.
112
- def resolve_targets(targets)
113
- targets.map { |target| resolve_target(target) }
114
- end
115
-
116
- # Splits a target into (path, line). RSpec example locations end in
117
- # `:LINE` (or `:LINE:LINE...` for nested groups), so the suffix is
118
- # anchored to the END of the string — that way path components which
119
- # themselves contain colons (e.g. a Windows-style `C:/proj/spec/x.rb`)
120
- # are not mis-split by a naive `partition(":")`.
121
- TARGET_LINE_SUFFIX = /\A(.+?)(:\d+(?::\d+)*)\z/
122
- private_constant :TARGET_LINE_SUFFIX
123
-
124
- def resolve_target(target)
125
- return target if target.start_with?("/")
126
-
127
- if (match = TARGET_LINE_SUFFIX.match(target))
128
- path = match[1]
129
- line_suffix = match[2]
130
- else
131
- path = target
132
- line_suffix = ""
133
- end
134
-
135
- return target if File.exist?(path)
136
- return target unless Evilution.in_isolated_worker?
137
-
138
- absolute = File.expand_path(path, Evilution::PROJECT_ROOT)
139
- return target unless File.exist?(absolute)
160
+ execute_run(args, command, ordered: ordered)
161
+ end
140
162
 
141
- "#{absolute}#{line_suffix}"
163
+ def run_args(order, files)
164
+ ["--format", "progress", "--no-color", "--order", order.to_s, *files]
142
165
  end
143
166
 
144
- def execute_run(args, command)
167
+ def execute_run(args, command, ordered: false)
145
168
  snapshot = @state_guard.snapshot
169
+ quit_before = ::RSpec.world.wants_to_quit
170
+ detector = nil
146
171
  begin
147
172
  out = StringIO.new
148
173
  err = StringIO.new
149
174
  claim_rspec_streams(out, err)
150
- detector = @crash_detector_lifecycle.register
151
- status = ::RSpec::Core::Runner.run(args, err, out)
152
- flagged = known_failures_only?(status)
153
- @result_builder.from_run(status, command, detector, examples_loaded:, known_failures_only: flagged)
175
+ detector = @crash_detector_lifecycle.register(@stop_policy)
176
+ status = @killer_first.during(ordered) { ::RSpec::Core::Runner.run(args, err, out) }
177
+ build_result(status, command, detector)
154
178
  rescue StandardError => e
155
179
  { passed: false, error: e.message, test_command: command }
156
180
  ensure
181
+ lower_own_stop(quit_before, detector)
157
182
  @state_guard.release(snapshot)
158
183
  end
159
184
  end
160
185
 
186
+ # Which example killed, if one did, and how many ran: the parent runs the
187
+ # killer first for the mutations next to this one, and the count is what
188
+ # the ordering saves.
189
+ def build_result(status, command, detector)
190
+ flagged = known_failures_only?(status)
191
+ result = @result_builder.from_run(status, command, detector, examples_loaded:, known_failures_only: flagged)
192
+ @result_builder.with_run_facts(result, killing_example: detector.killing_id,
193
+ examples_run: ExampleIds.executed_count(::RSpec.world))
194
+ end
195
+
196
+ # A run that stops at its kill raises RSpec.world.wants_to_quit, and left up
197
+ # it would make the next RSpec::Core::Runner.run in this process return a
198
+ # failure status without running anything -- a false kill. So the flag goes
199
+ # back to what it was, but only when the crash detector raised it: raised by
200
+ # anything else, such as RSpec's own handler on the user's first Ctrl-C, it
201
+ # stays up even if the run later stopped at a kill, so the next Ctrl-C still
202
+ # exits.
203
+ def lower_own_stop(quit_before, detector)
204
+ return if detector.nil?
205
+
206
+ ::RSpec.world.wants_to_quit = quit_before if detector.raised_quit?
207
+ end
208
+
161
209
  # RSpec redirects its own output to the streams it is handed only when the
162
210
  # stream its configuration already holds is the current `$stdout`. In-process
163
211
  # isolation swaps `$stdout` for a null IO around every mutation, and a
@@ -212,6 +260,9 @@ end
212
260
 
213
261
  require_relative "rspec/framework_loader"
214
262
  require_relative "rspec/example_ids"
263
+ require_relative "rspec/example_order"
264
+ require_relative "rspec/killer_first"
265
+ require_relative "rspec/target_resolver"
215
266
  require_relative "rspec/test_file_resolver"
216
267
  require_relative "rspec/unresolved_spec_warner"
217
268
  require_relative "rspec/example_filter_applier"