evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../integration"
4
+ require_relative "known_failures"
5
+ require_relative "../infra_crash"
6
+
7
+ # Decides when a mutation's test run has seen enough to stop.
8
+ #
9
+ # A failure is decisive when it says something about the mutation: a test the
10
+ # baseline did not already see failing, and not a crash of the test
11
+ # infrastructure itself (which would say nothing whatever the mutation was).
12
+ # The first decisive failure latches the policy, and the integration stops its
13
+ # framework.
14
+ class Evilution::Integration::StopPolicy
15
+ def initialize(enabled:, known_failures:)
16
+ @enabled = enabled
17
+ @known_failures = known_failures
18
+ @stopped = false
19
+ end
20
+
21
+ # exception is nil for an assertion failure; a crash passes its exception.
22
+ def decisive?(id:, exception: nil)
23
+ return false if @known_failures.include?(id)
24
+ return true if exception.nil?
25
+
26
+ !Evilution::InfraCrash.class_name?(exception.class.name)
27
+ end
28
+
29
+ # Returns true, and latches, on each decisive failure recorded while
30
+ # enabled; false otherwise.
31
+ def record(id:, exception: nil)
32
+ return false unless @enabled
33
+ return false unless decisive?(id: id, exception: exception)
34
+
35
+ @stopped = true
36
+ end
37
+
38
+ def stopped?
39
+ @stopped
40
+ end
41
+
42
+ def reset
43
+ @stopped = false if @enabled
44
+ end
45
+
46
+ # Null object for integrations given no policy. Frozen and shared, so it
47
+ # never touches state: record and reset are no-ops.
48
+ DISABLED = new(enabled: false, known_failures: Evilution::Integration::KnownFailures.new([])).freeze
49
+ end
@@ -11,16 +11,35 @@ require_relative "../test_unit"
11
11
  module Evilution::Integration::TestUnit::Dispatcher
12
12
  module_function
13
13
 
14
- def call(test_case_classes, name: "evilution")
14
+ # on_fault, when given, is called with each fault the run records and the
15
+ # run's Test::Unit::TestResult, which it may stop. The runner returns that
16
+ # result whether or not the run was stopped.
17
+ def call(test_case_classes, name: "evilution", on_fault: nil)
15
18
  require "test/unit/ui/console/testrunner"
16
19
  suite = ::Test::Unit::TestSuite.new(name)
17
20
  test_case_classes.each { |klass| suite << klass.suite }
18
21
  out = StringIO.new
19
22
  runner = ::Test::Unit::UI::Console::TestRunner.new(suite, output: out)
23
+ runner.listeners << FaultListener.new(on_fault) unless on_fault.nil?
20
24
  runner.start
21
25
  end
22
26
 
23
27
  def test_method_count(test_case_classes)
24
28
  test_case_classes.sum { |klass| klass.suite.tests.length }
25
29
  end
30
+
31
+ # Attached to the runner's mediator. The run's result exists only once the
32
+ # mediator has started it, and `result.stop` works only from then on: the
33
+ # mediator sets the tag it throws to just before announcing the start.
34
+ class FaultListener
35
+ def initialize(on_fault)
36
+ @on_fault = on_fault
37
+ end
38
+
39
+ def attach_to_mediator(mediator)
40
+ mediator.add_listener(::Test::Unit::UI::TestRunnerMediator::STARTED) do |result|
41
+ result.add_listener(::Test::Unit::TestResult::FAULT) { |fault| @on_fault.call(fault, result) }
42
+ end
43
+ end
44
+ end
26
45
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "../test_unit"
4
+ require_relative "../../class_tree"
4
5
 
5
6
  # Tracks Test::Unit::TestCase descendants in the host process. Test::Unit has
6
7
  # no public registry-clear method analogous to Minitest::Runnable.runnables;
@@ -8,13 +9,17 @@ require_relative "../test_unit"
8
9
  # round by diffing the descendant set before and after #load. Keeping that
9
10
  # responsibility in its own object makes it cheap to stub in tests and lets
10
11
  # the integration's main class read as orchestration.
12
+ #
13
+ # The descendants come from the class tree, not from a walk of the heap: a
14
+ # mutation process loads its test files afresh, and walking every object
15
+ # before and after each load is a cost per mutation.
11
16
  module Evilution::Integration::TestUnit::SubjectClassRegistry
12
17
  module_function
13
18
 
14
19
  def descendants
15
20
  return [] unless defined?(::Test::Unit::TestCase)
16
21
 
17
- ObjectSpace.each_object(Class).select { |c| c < ::Test::Unit::TestCase }
22
+ Evilution::ClassTree.descendants(::Test::Unit::TestCase)
18
23
  end
19
24
 
20
25
  # Yields, captures the descendant set before/after, and returns the diff.
@@ -3,6 +3,7 @@
3
3
  require_relative "base"
4
4
  require_relative "test_unit_crash_detector"
5
5
  require_relative "known_failures"
6
+ require_relative "stop_policy"
6
7
  require_relative "loading/test_load_path"
7
8
  require_relative "loading/test_class_cache"
8
9
  require_relative "../spec_resolver"
@@ -76,7 +77,8 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
76
77
  end
77
78
 
78
79
  # known_failures: the ids of the tests the baseline saw failing (see TestIds).
79
- def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil, known_failures: [])
80
+ def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil, known_failures: [],
81
+ stop_on_kill: false)
80
82
  require_relative "test_unit/framework_loader"
81
83
  require_relative "test_unit/subject_class_registry"
82
84
  require_relative "test_unit/dispatcher"
@@ -84,6 +86,7 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
84
86
  require_relative "test_unit/result_builder"
85
87
  require_relative "test_unit/test_ids"
86
88
  @known_failures = Evilution::Integration::KnownFailures.new(known_failures)
89
+ @stop_policy = Evilution::Integration::StopPolicy.new(enabled: stop_on_kill, known_failures: @known_failures)
87
90
  @framework_loader = FrameworkLoader.new
88
91
  @file_resolver = TestFileResolver.new(
89
92
  test_files: test_files,
@@ -121,7 +124,10 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
121
124
  return ResultBuilder.no_tests_ran(command) if Dispatcher.test_method_count(new_classes).zero?
122
125
 
123
126
  detector = reset_crash_detector
124
- result = Dispatcher.call(new_classes, name: "evilution-mutation")
127
+ # Reset: a policy latched by the previous run would stop this one at its
128
+ # first failure, decisive or not.
129
+ @stop_policy.reset
130
+ result = Dispatcher.call(new_classes, name: "evilution-mutation", on_fault: method(:stop_at_decisive_fault))
125
131
  # Before the crash check: a test that was already failing may fail by
126
132
  # raising now, and a crash in it is no more a kill than its failure was.
127
133
  return ResultBuilder.known_failures_only(command) if @known_failures.only?(TestIds.failed(result))
@@ -130,6 +136,26 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
130
136
  ResultBuilder.call(passed: result.passed?, command: command, detector: detector)
131
137
  end
132
138
 
139
+ # Only failures and errors decide a run (see TestIds.faults); pendings,
140
+ # omissions and notifications are faults too, and are passed over. The run
141
+ # is stopped once the failing test has finished, not from inside its
142
+ # failure, so that the result counts it as run.
143
+ def stop_at_decisive_fault(fault, result)
144
+ return unless decisive_fault?(fault)
145
+
146
+ result.add_listener(::Test::Unit::TestResult::FINISHED) { result.stop }
147
+ end
148
+
149
+ def decisive_fault?(fault)
150
+ if fault.is_a?(::Test::Unit::Error)
151
+ @stop_policy.record(id: fault.test_name, exception: fault.exception)
152
+ elsif fault.is_a?(::Test::Unit::Failure)
153
+ @stop_policy.record(id: fault.test_name)
154
+ else
155
+ false
156
+ end
157
+ end
158
+
133
159
  def load_test_classes(files)
134
160
  Evilution::Integration::Loading::TestLoadPath.add!(files)
135
161
  paths = files.map { |f| File.expand_path(f, Evilution.project_base_dir) }
@@ -13,6 +13,11 @@ require_relative "../isolation"
13
13
  class Evilution::Isolation::Fork
14
14
  GRACE_PERIOD = 2
15
15
  REAP_DEADLINE = 1.0
16
+ # Bounds of one wait between liveness checks while reaping. A child that has
17
+ # written its result exits within a millisecond or two, so the first waits
18
+ # are short; they double towards the cap for one that takes longer.
19
+ REAP_STEP_MIN = 0.001
20
+ REAP_STEP_MAX = 0.05
16
21
 
17
22
  def initialize(hooks: nil)
18
23
  @hooks = hooks
@@ -160,9 +165,14 @@ class Evilution::Isolation::Fork
160
165
  end
161
166
  end
162
167
 
168
+ # The child does not evaluate the original source again after the tests, so
169
+ # the RSS reader is taken as a method object before they run: a mutation of
170
+ # its file would otherwise measure (or raise) on the mutant's behalf. The
171
+ # reader itself is taken, not `rss_kb`, whose body would look it up again.
163
172
  def execute_in_child(mutation, test_command)
173
+ rss_kb_for = Evilution::Memory.method(:rss_kb_for)
164
174
  result = test_command.call(mutation)
165
- { child_rss_kb: Evilution::Memory.rss_kb }.merge(result)
175
+ { child_rss_kb: rss_kb_for.call(Process.pid) }.merge(result)
166
176
  rescue ScriptError, StandardError => e
167
177
  {
168
178
  passed: false,
@@ -186,7 +196,7 @@ class Evilution::Isolation::Fork
186
196
 
187
197
  if read_io.wait_readable([remaining, 0.5].min)
188
198
  payload = read_payload(read_io, deadline)
189
- return reap_and_decode(handle, payload) if payload
199
+ return reap_and_decode(handle, payload, read_io) if payload
190
200
  end
191
201
 
192
202
  next unless @supervisor.reap_nonblock(handle)
@@ -206,22 +216,46 @@ class Evilution::Isolation::Fork
206
216
  # in execute_in_child waiting on a subject grandchild the mutation broke.
207
217
  # wait_for_result has already returned by this point, so the per-mutation
208
218
  # timeout cannot fire. Bound the wait and fall back to the TERM/KILL ladder.
209
- def reap_and_decode(handle, payload)
219
+ #
220
+ # The child closes its end of the pipe right before it exits, so the wait is
221
+ # on the pipe: it ends the moment the child lets go of it, not at the end of
222
+ # a fixed step. A grandchild holding the pipe keeps it open past the child's
223
+ # exit, so the waits are sliced and the child's liveness checked after each.
224
+ def reap_and_decode(handle, payload, read_io)
210
225
  deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + REAP_DEADLINE
211
- loop do
212
- break if @supervisor.reap_nonblock(handle)
226
+ step = REAP_STEP_MIN
227
+ pipe_open = true
228
+ until @supervisor.reap_nonblock(handle)
229
+ remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
230
+ return terminate_after_reap_deadline(handle, payload) if remaining <= 0
213
231
 
214
- if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
215
- @supervisor.terminate(handle, grace: GRACE_PERIOD)
216
- # The status now records OUR signal, so it cannot be reported as the
217
- # child's own cause of death.
218
- return decode_payload(payload, handle, terminated: true)
219
- end
220
- sleep 0.05
232
+ pipe_open = wait_for_pipe_close(read_io, [remaining, step].min, pipe_open)
233
+ step = [step * 2, REAP_STEP_MAX].min
221
234
  end
222
235
  decode_payload(payload, handle)
223
236
  end
224
237
 
238
+ def terminate_after_reap_deadline(handle, payload)
239
+ @supervisor.terminate(handle, grace: GRACE_PERIOD)
240
+ # The status now records OUR signal, so it cannot be reported as the
241
+ # child's own cause of death.
242
+ decode_payload(payload, handle, terminated: true)
243
+ end
244
+
245
+ # Waits up to `seconds` for every writer to close the pipe and returns
246
+ # whether it is still open. Once it has closed it stays readable, so from
247
+ # then on the wait is a plain pause; bytes written after the result are
248
+ # discarded.
249
+ def wait_for_pipe_close(read_io, seconds, pipe_open)
250
+ unless pipe_open
251
+ sleep(seconds)
252
+ return false
253
+ end
254
+ return true unless read_io.wait_readable(seconds)
255
+
256
+ !read_io.read_nonblock(4096, exception: false).nil?
257
+ end
258
+
225
259
  def read_payload(read_io, deadline)
226
260
  header = read_n_bytes(read_io, 4, deadline)
227
261
  return nil unless header
@@ -336,7 +370,18 @@ class Evilution::Isolation::Fork
336
370
  klass: result[:error_class],
337
371
  backtrace: result[:error_backtrace]
338
372
  ),
339
- known_failures_only: result[:known_failures_only]
373
+ known_failures_only: result[:known_failures_only],
374
+ timings: mutation_timings(result, duration),
375
+ killing_test: result[:killing_example],
376
+ examples_run: result[:examples_run]
340
377
  )
341
378
  end
379
+
380
+ # A timed-out run never reports its own timings. Applying a mutation takes
381
+ # a fraction of a second, so its time is counted as the test run's.
382
+ def mutation_timings(result, duration)
383
+ return Evilution::Result::MutationTimings.new(test_run: duration) if result[:timeout]
384
+
385
+ Evilution::Result::MutationTimings.from_h(result[:timings])
386
+ end
342
387
  end
@@ -198,7 +198,18 @@ class Evilution::Isolation::InProcess
198
198
  klass: result[:error_class],
199
199
  backtrace: result[:error_backtrace]
200
200
  ),
201
- known_failures_only: result[:known_failures_only]
201
+ known_failures_only: result[:known_failures_only],
202
+ timings: mutation_timings(result, duration),
203
+ killing_test: result[:killing_example],
204
+ examples_run: result[:examples_run]
202
205
  )
203
206
  end
207
+
208
+ # A timed-out run never reports its own timings. Applying a mutation takes
209
+ # a fraction of a second, so its time is counted as the test run's.
210
+ def mutation_timings(result, duration)
211
+ return Evilution::Result::MutationTimings.new(test_run: duration) if result[:timeout]
212
+
213
+ Evilution::Result::MutationTimings.from_h(result[:timings])
214
+ end
204
215
  end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "version"
4
+
5
+ # The lines of a file a run is limited to, as any number of line ranges: the
6
+ # one range given on the command line (lib/foo.rb:15-30), or each hunk git
7
+ # reports as changed. Ranges are kept sorted, with overlapping and touching
8
+ # ones merged; an open-ended range ends in Float::INFINITY.
9
+ class Evilution::LineSet
10
+ attr_reader :ranges
11
+
12
+ def self.wrap(lines)
13
+ lines.is_a?(self) ? lines : new([lines])
14
+ end
15
+
16
+ def initialize(ranges)
17
+ @ranges = merge(ranges.reject { |range| range.last < range.first }).freeze
18
+ freeze
19
+ end
20
+
21
+ def cover?(line)
22
+ ranges.any? { |range| range.cover?(line) }
23
+ end
24
+
25
+ # Whether any line from first to last is in the set.
26
+ def overlap?(first, last)
27
+ ranges.any? { |range| first <= range.last && last >= range.first }
28
+ end
29
+
30
+ def &(other)
31
+ pairs = ranges.product(self.class.wrap(other).ranges)
32
+ self.class.new(pairs.map { |mine, theirs| [mine.first, theirs.first].max..[mine.last, theirs.last].min })
33
+ end
34
+
35
+ def empty?
36
+ ranges.empty?
37
+ end
38
+
39
+ def ==(other)
40
+ other.is_a?(self.class) && ranges == other.ranges
41
+ end
42
+ alias eql? ==
43
+
44
+ def hash
45
+ ranges.hash
46
+ end
47
+
48
+ # Spelled the way a range is passed on the command line: `2-4,9`, `3-`.
49
+ def to_s
50
+ ranges.map { |range| label(range) }.join(",")
51
+ end
52
+
53
+ private
54
+
55
+ def merge(ranges)
56
+ ranges.sort_by(&:first).each_with_object([]) do |range, merged|
57
+ last = merged.last
58
+ if last && range.first <= last.last + 1
59
+ merged[-1] = last.first..[last.last, range.last].max
60
+ else
61
+ merged << range
62
+ end
63
+ end
64
+ end
65
+
66
+ def label(range)
67
+ return "#{range.first}-" if range.last == Float::INFINITY
68
+ return range.first.to_s if range.first == range.last
69
+
70
+ "#{range.first}-#{range.last}"
71
+ end
72
+ end
@@ -6,6 +6,14 @@ require_relative "../../../config"
6
6
  require_relative "../../../version"
7
7
 
8
8
  class Evilution::MCP::InfoTool::Actions::Environment < Evilution::MCP::InfoTool::Actions::Base
9
+ # The config readers reported under "settings", each under its own name.
10
+ SETTINGS = %w[
11
+ timeout format integration jobs isolation baseline incremental fail_fast stop_on_kill killer_first
12
+ skip_uncovered probe_first min_score suggest_tests save_session target skip_heredoc_literals ignore_patterns
13
+ skip_sites timeout_factor timeout_floor timeout_ceiling copy_worker_databases
14
+ ].freeze
15
+ private_constant :SETTINGS
16
+
9
17
  def self.call(**)
10
18
  config = Evilution::Config.new(skip_config_file: false)
11
19
  config_file = Evilution::Config::CONFIG_FILES.find { |path| File.exist?(path) }
@@ -21,23 +29,11 @@ class Evilution::MCP::InfoTool::Actions::Environment < Evilution::MCP::InfoTool:
21
29
  class << self
22
30
  private
23
31
 
32
+ # With no timeout set, "timeout" is only the fallback: each mutation's limit
33
+ # follows the baseline (timeout_factor, timeout_floor, timeout_ceiling).
24
34
  def settings(config)
25
- {
26
- "timeout" => config.timeout,
27
- "format" => config.format,
28
- "integration" => config.integration,
29
- "jobs" => config.jobs,
30
- "isolation" => config.isolation,
31
- "baseline" => config.baseline,
32
- "incremental" => config.incremental,
33
- "fail_fast" => config.fail_fast,
34
- "min_score" => config.min_score,
35
- "suggest_tests" => config.suggest_tests,
36
- "save_session" => config.save_session,
37
- "target" => config.target,
38
- "skip_heredoc_literals" => config.skip_heredoc_literals,
39
- "ignore_patterns" => config.ignore_patterns
40
- }
35
+ SETTINGS.to_h { |name| [name, config.public_send(name)] }
36
+ .merge("adaptive_timeout" => config.adaptive_timeout?)
41
37
  end
42
38
  end
43
39
  end
@@ -5,7 +5,7 @@ require_relative "base"
5
5
  require_relative "../config_factory"
6
6
  require_relative "../../../runner"
7
7
  require_relative "../../../mutator/registry"
8
- require_relative "../../../ast/pattern/filter"
8
+ require_relative "../../../ast/skip_filter"
9
9
 
10
10
  class Evilution::MCP::InfoTool::Actions::Subjects < Evilution::MCP::InfoTool::Actions::Base
11
11
  def self.call(files: nil, line_ranges: nil, target: nil, integration: nil, skip_config: nil, **)
@@ -30,7 +30,7 @@ class Evilution::MCP::InfoTool::Actions::Subjects < Evilution::MCP::InfoTool::Ac
30
30
  def subject_entries(subjects, config)
31
31
  registry = Evilution::Mutator::Registry.default
32
32
  filter = build_subject_filter(config)
33
- operator_options = { skip_heredoc_literals: config.skip_heredoc_literals? }
33
+ operator_options = { skip_heredoc_literals: config.skip_heredoc_literals?, line_ranges: config.line_ranges }
34
34
 
35
35
  subjects.map do |subj|
36
36
  count = registry.mutations_for(subj, filter: filter, operator_options: operator_options).length
@@ -49,9 +49,7 @@ class Evilution::MCP::InfoTool::Actions::Subjects < Evilution::MCP::InfoTool::Ac
49
49
  end
50
50
 
51
51
  def build_subject_filter(config)
52
- return nil if config.ignore_patterns.empty?
53
-
54
- Evilution::AST::Pattern::Filter.new(config.ignore_patterns)
52
+ Evilution::AST::SkipFilter.for(config)
55
53
  end
56
54
  end
57
55
  end
@@ -18,12 +18,17 @@ module Evilution::MCP::InfoTool::StatusGlossary
18
18
  {
19
19
  "status" => "survived",
20
20
  "meaning" => "No test failed when the mutation was applied — gap in coverage. " \
21
- "The test suite did not detect the behavioral change.",
21
+ "The test suite did not detect the behavioral change. With probe_first, a survivor " \
22
+ "carrying implied_by was not run: the subject's probe mutation (its body replaced " \
23
+ "by a raise) failed no test, so no test reaches this code.",
22
24
  "counted_in_score" => true
23
25
  },
24
26
  {
25
27
  "status" => "timeout",
26
- "meaning" => "Test run exceeded the configured per-mutation timeout. " \
28
+ "meaning" => "Test run exceeded the per-mutation time limit (time_limit, in seconds). " \
29
+ "Left unset, the limit adapts to the baseline time of the mutation's spec files; " \
30
+ "a timeout under a limit shorter than 30 seconds is re-run under 30 seconds, " \
31
+ "and that run decides. " \
27
32
  "Treated like survived for scoring (counted in the denominator); " \
28
33
  "may indicate an infinite loop introduced by the mutation.",
29
34
  "counted_in_score" => true
@@ -60,6 +65,14 @@ module Evilution::MCP::InfoTool::StatusGlossary
60
65
  "The file has no corresponding test file the resolver could locate.",
61
66
  "counted_in_score" => false
62
67
  },
68
+ {
69
+ "status" => "uncovered",
70
+ "meaning" => "No example executes any line the mutation changes, so it was never run: with " \
71
+ "coverage targeting, the coverage map shows the line never ran, neither in an example " \
72
+ "nor while the files loaded. Untested code. Counted in the score as a survivor, and " \
73
+ "listed apart from the survivors that did run.",
74
+ "counted_in_score" => true
75
+ },
63
76
  {
64
77
  "status" => "unparseable",
65
78
  "meaning" => "Mutated source failed to parse (e.g. dangling heredoc after method_body_replacement). " \
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "json"
4
4
  require "mcp"
5
+ require_relative "../mcp"
5
6
  require_relative "../config"
6
7
  require_relative "../runner"
7
8
  require_relative "../mutator/registry"
@@ -10,8 +11,6 @@ require_relative "../spec_resolver"
10
11
  require_relative "../ast/pattern/filter"
11
12
  require_relative "../version"
12
13
 
13
- require_relative "../mcp"
14
-
15
14
  class Evilution::MCP::InfoTool < MCP::Tool
16
15
  VALID_ACTIONS = %w[subjects tests environment statuses feedback].freeze
17
16
 
@@ -75,7 +74,9 @@ class Evilution::MCP::InfoTool < MCP::Tool
75
74
  class << self
76
75
  def call(server_context:, action: nil, files: nil, target: nil, spec: nil, integration: nil, skip_config: nil)
77
76
  return ResponseFormatter.error("config_error", "action is required") unless action
78
- return ResponseFormatter.error("config_error", "unknown action: #{action}") unless ACTIONS.key?(action)
77
+
78
+ action_class = actions[action]
79
+ return ResponseFormatter.error("config_error", "unknown action: #{action}") unless action_class
79
80
 
80
81
  parsed_files = nil
81
82
  line_ranges = nil
@@ -85,13 +86,32 @@ class Evilution::MCP::InfoTool < MCP::Tool
85
86
  line_ranges = parsed.ranges
86
87
  end
87
88
 
88
- ACTIONS[action].call(
89
+ action_class.call(
89
90
  files: parsed_files, line_ranges: line_ranges, target: target, spec: spec,
90
91
  integration: integration, skip_config: skip_config
91
92
  )
92
93
  rescue Evilution::Error => e
93
94
  ResponseFormatter.error_for(e)
94
95
  end
96
+
97
+ private
98
+
99
+ # Built when a request is dispatched, not while this file loads: each
100
+ # action file requires this file for its namespace, so when an action file
101
+ # is the first one required, its class is only defined after this file
102
+ # has finished loading.
103
+ #
104
+ # Its keys must equal VALID_ACTIONS; both are literals, so a spec checks
105
+ # that rather than every request.
106
+ def actions
107
+ {
108
+ "subjects" => Actions::Subjects,
109
+ "tests" => Actions::Tests,
110
+ "environment" => Actions::Environment,
111
+ "statuses" => Actions::Statuses,
112
+ "feedback" => Actions::Feedback
113
+ }.freeze
114
+ end
95
115
  end
96
116
  end
97
117
 
@@ -107,16 +127,3 @@ require_relative "info_tool/actions/tests"
107
127
  require_relative "info_tool/actions/environment"
108
128
  require_relative "info_tool/actions/statuses"
109
129
  require_relative "info_tool/actions/feedback"
110
-
111
- Evilution::MCP::InfoTool.const_set(:ACTIONS, {
112
- "subjects" => Evilution::MCP::InfoTool::Actions::Subjects,
113
- "tests" => Evilution::MCP::InfoTool::Actions::Tests,
114
- "environment" => Evilution::MCP::InfoTool::Actions::Environment,
115
- "statuses" => Evilution::MCP::InfoTool::Actions::Statuses,
116
- "feedback" => Evilution::MCP::InfoTool::Actions::Feedback
117
- }.freeze)
118
- Evilution::MCP::InfoTool.send(:private_constant, :ACTIONS)
119
-
120
- unless Evilution::MCP::InfoTool.send(:const_get, :ACTIONS).keys == Evilution::MCP::InfoTool::VALID_ACTIONS
121
- raise "InfoTool action drift: ACTIONS keys do not match VALID_ACTIONS"
122
- end
@@ -3,13 +3,14 @@
3
3
  require_relative "../mutate_tool"
4
4
  require_relative "option_parser"
5
5
  require_relative "../../config"
6
+ require_relative "../../git/since_scope"
6
7
 
7
8
  module Evilution::MCP::MutateTool::ConfigBuilder
8
9
  def self.build(files:, line_ranges:, params:)
9
10
  # Preload is disabled for MCP invocations: `require`-ing Rails into the
10
11
  # long-lived MCP server would poison subsequent runs against other
11
12
  # projects. MCP users who want the speedup should use the CLI.
12
- opts = { target_files: files, line_ranges: line_ranges, format: :json, quiet: true, preload: false }
13
+ opts = scope(files, line_ranges, params).merge(format: :json, quiet: true, preload: false)
13
14
  opts[:skip_config_file] = true if params[:skip_config]
14
15
  opts[:spec_files] = params[:spec] if params[:spec]
15
16
  Evilution::MCP::MutateTool::OptionParser::PASSTHROUGH_KEYS.each do |key|
@@ -17,4 +18,15 @@ module Evilution::MCP::MutateTool::ConfigBuilder
17
18
  end
18
19
  Evilution::Config.new(**opts)
19
20
  end
21
+
22
+ # The files and line ranges, limited to the lines changed since the ref
23
+ # given as since, which is checked before git reads it.
24
+ def self.scope(files, line_ranges, params)
25
+ since = Evilution::Config::Validators::Since.call(params[:since], target: params[:target])
26
+ paths = Evilution::Git::SinceScope.apply(since, files: files, line_ranges: line_ranges)
27
+ opts = { target_files: paths.files, line_ranges: paths.ranges }
28
+ opts[:since] = since if since
29
+ opts
30
+ end
31
+ private_class_method :scope
20
32
  end
@@ -5,8 +5,8 @@ require_relative "../mutate_tool"
5
5
  module Evilution::MCP::MutateTool::OptionParser
6
6
  VALID_VERBOSITIES = %w[full summary minimal].freeze
7
7
  PASSTHROUGH_KEYS = %i[target timeout jobs fail_fast suggest_tests incremental integration
8
- isolation baseline save_session preload].freeze
9
- ALLOWED_OPT_KEYS = (PASSTHROUGH_KEYS + %i[spec skip_config]).freeze
8
+ isolation baseline save_session preload stop_on_kill skip_sites].freeze
9
+ ALLOWED_OPT_KEYS = (PASSTHROUGH_KEYS + %i[spec skip_config since]).freeze
10
10
 
11
11
  ParsedPaths = Data.define(:files, :ranges)
12
12
 
@@ -14,7 +14,7 @@ module Evilution::MCP::MutateTool::ProgressStreamer
14
14
  disabled = false
15
15
 
16
16
  proc do |result|
17
- next unless result.survived?
17
+ next unless result.survived? || result.uncovered?
18
18
  next if disabled
19
19
 
20
20
  begin