evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prism"
4
+ require_relative "../skip_sites"
5
+
6
+ # Whether evaluating an expression only reads: a mutation inside it can then
7
+ # change the value it builds, and nothing else. Literals, variable and
8
+ # constant reads, interpolation and the control flow between them read; so
9
+ # do the calls known to only compute a value: operators, `[]`, `format`,
10
+ # `join`, `to_s(base)` whatever their arguments, and, without arguments or a
11
+ # block, predicates (`empty?`) and common readers (`id`, `name`, `message`,
12
+ # `inspect`, `size`, `to_i`). A call's name alone says nothing of what it
13
+ # does -- `queue.pop`, `io.read`, `enum.next` change state -- so any other
14
+ # call, an assignment, `yield`, a backtick may change state, and mutating it
15
+ # could change more than a message.
16
+ module Evilution::AST::SkipSites::Purity
17
+ READS = [
18
+ Prism::StringNode, Prism::InterpolatedStringNode, Prism::EmbeddedStatementsNode, Prism::EmbeddedVariableNode,
19
+ Prism::SymbolNode, Prism::InterpolatedSymbolNode, Prism::RegularExpressionNode,
20
+ Prism::IntegerNode, Prism::FloatNode, Prism::RationalNode, Prism::ImaginaryNode,
21
+ Prism::NilNode, Prism::TrueNode, Prism::FalseNode, Prism::SelfNode,
22
+ Prism::SourceFileNode, Prism::SourceLineNode, Prism::SourceEncodingNode,
23
+ Prism::LocalVariableReadNode, Prism::InstanceVariableReadNode, Prism::ClassVariableReadNode,
24
+ Prism::GlobalVariableReadNode, Prism::BackReferenceReadNode, Prism::NumberedReferenceReadNode,
25
+ Prism::ConstantReadNode, Prism::ConstantPathNode,
26
+ Prism::ArgumentsNode, Prism::StatementsNode, Prism::ParenthesesNode,
27
+ Prism::ArrayNode, Prism::HashNode, Prism::KeywordHashNode, Prism::AssocNode, Prism::RangeNode,
28
+ Prism::AndNode, Prism::OrNode, Prism::IfNode, Prism::UnlessNode, Prism::ElseNode, Prism::DefinedNode,
29
+ Prism::BlockNode, Prism::BlockParametersNode, Prism::ParametersNode, Prism::RequiredParameterNode
30
+ ].freeze
31
+
32
+ COMPUTING_CALLS = %i[
33
+ [] + - * / % ** == != < > <= >= <=> ! to_s join format sprintf
34
+ ].freeze
35
+
36
+ READERS = %i[
37
+ -@ +@ ~ class name message full_message inspect to_str to_sym to_i to_f to_a to_h id object_id hash
38
+ size length count first last keys values key value path basename dirname extname upcase downcase strip itself
39
+ ].freeze
40
+
41
+ private_constant :READS, :COMPUTING_CALLS, :READERS
42
+
43
+ module_function
44
+
45
+ def pure?(root)
46
+ pending = [root]
47
+ until pending.empty?
48
+ node = pending.pop
49
+ return false unless reads?(node)
50
+
51
+ pending.concat(node.compact_child_nodes)
52
+ end
53
+ true
54
+ end
55
+
56
+ def reads?(node)
57
+ return reading_call?(node) if node.is_a?(Prism::CallNode)
58
+
59
+ READS.any? { |type| node.is_a?(type) }
60
+ end
61
+
62
+ # A block, even `&:sym`, runs code the call decides when to run.
63
+ def reading_call?(call)
64
+ return false if call.block
65
+
66
+ name = call.name
67
+ COMPUTING_CALLS.include?(name) || (call.arguments.nil? && (READERS.include?(name) || name.end_with?("?")))
68
+ end
69
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prism"
4
+ require_relative "../skip_sites"
5
+ require_relative "nodes"
6
+
7
+ # `require` and `require_relative` that run while the file loads: at the top
8
+ # of the file or in a class or module body, sent to no receiver or to
9
+ # Kernel. By the time a mutant is evaluated the file has been loaded once
10
+ # already, so dropping or changing such a require changes nothing the tests
11
+ # can see, and the mutant survives for no reason worth reporting. A require
12
+ # inside a method, block or lambda loads lazily and is left alone: it runs
13
+ # when the code under test runs. A condition guarding a require is not part
14
+ # of the site, nor is a require whose value is used (`HAS_OJ = require
15
+ # "oj"`): dropping it changes the value.
16
+ #
17
+ # Every mutation inside a site is skipped, the call itself included.
18
+ class Evilution::AST::SkipSites::Requires
19
+ REQUIRES = %i[require require_relative].freeze
20
+ DEFERRED = [Prism::DefNode, Prism::BlockNode, Prism::LambdaNode].freeze
21
+ private_constant :REQUIRES, :DEFERRED
22
+
23
+ def name
24
+ :requires
25
+ end
26
+
27
+ # chain: the mutated node, then its ancestors, nearest first.
28
+ def covers?(chain)
29
+ chain.each_with_index.any? do |node, index|
30
+ Evilution::AST::SkipSites::Nodes.kernel_call?(node, REQUIRES) && runs_on_load?(chain, index) &&
31
+ Evilution::AST::SkipSites::Nodes.value_dropped?(chain, index)
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def runs_on_load?(chain, site_index)
38
+ chain.drop(site_index + 1).none? { |ancestor| DEFERRED.any? { |type| ancestor.is_a?(type) } }
39
+ end
40
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ast"
4
+
5
+ # Named presets of code whose mutations are rarely worth a test run: log
6
+ # lines, debug output, the messages of raised errors, load-time requires.
7
+ # Each preset is a structural rule over the Prism tree (see SkipSites::Presets).
8
+ module Evilution::AST::SkipSites
9
+ end
@@ -4,7 +4,9 @@ require_relative "../ast"
4
4
  require_relative "syntax_check"
5
5
 
6
6
  module Evilution::AST::SourceSurgeon
7
- Result = Struct.new(:source, :status, keyword_init: true) do
7
+ # region: what proved the source parses without it being parsed whole (see
8
+ # .check); nil when it was.
9
+ Result = Struct.new(:source, :status, :region, keyword_init: true) do
8
10
  def ok?
9
11
  status == :ok
10
12
  end
@@ -15,9 +17,25 @@ module Evilution::AST::SourceSurgeon
15
17
  end
16
18
 
17
19
  def self.apply(source, offset:, length:, replacement:)
20
+ check(splice(source, offset: offset, length: length, replacement: replacement))
21
+ end
22
+
23
+ # The source with the bytes replaced, not yet checked: cheap next to the
24
+ # parse that checking takes.
25
+ def self.splice(source, offset:, length:, replacement:)
18
26
  binary = source.b
19
27
  binary[offset, length] = replacement.b
20
- mutated = binary.force_encoding(source.encoding)
28
+ binary.force_encoding(source.encoding)
29
+ end
30
+
31
+ # The block, when given, can prove the source parses for less than
32
+ # parsing it whole, as AST::DefRegion#parse does for a mutation inside a
33
+ # method. What it returns stands for the parse; when it is nil, the whole
34
+ # source is parsed.
35
+ def self.check(mutated)
36
+ region = yield if block_given?
37
+ return Result.new(source: mutated, status: :ok, region: region).freeze if region
38
+
21
39
  status = Evilution::AST::SyntaxCheck.valid?(mutated) ? :ok : :unparseable
22
40
  Result.new(source: mutated, status: status).freeze
23
41
  end
@@ -7,12 +7,12 @@ require_relative "../ast"
7
7
  # Answers whether mutated source is Ruby the running interpreter will accept.
8
8
  #
9
9
  # Mutations are built and screened with Prism, but Prism and the older
10
- # parse.y parser do not agree on everything: Prism takes a bare constant
11
- # operator write inside a method (`def f; LIMIT ||= 1; end`) and a `nil[...]`
12
- # pattern, both of which parse.y rejects. An interpreter that compiles with
13
- # parse.y -- Ruby 3.3 by default, any later one started with
14
- # `--parser=parse.y` -- would be handed a mutant marked parseable and raise
15
- # SyntaxError evaluating it.
10
+ # parse.y parser do not agree on everything: Prism takes a `nil[...]`
11
+ # pattern, and before 2.0 also a bare constant operator write inside a method
12
+ # (`def f; LIMIT ||= 1; end`), both of which parse.y rejects. An interpreter
13
+ # that compiles with parse.y -- Ruby 3.3 by default, any later one started
14
+ # with `--parser=parse.y` -- would be handed a mutant marked parseable and
15
+ # raise SyntaxError evaluating it.
16
16
  #
17
17
  # So Prism decides first, and where the interpreter does not compile with
18
18
  # Prism, parse.y is asked as well. That second parse costs about what the
@@ -21,8 +21,10 @@ module Evilution::AST::SyntaxCheck
21
21
  # How the interpreter says it compiles: `ruby 3.4.1 (...) +PRISM [...]`.
22
22
  PRISM_INTERPRETER = RUBY_DESCRIPTION.include?("+PRISM")
23
23
 
24
+ # Prism is only asked whether the source parses, which builds no syntax
25
+ # tree: the answer is the same, without the tree a mutant has no use for.
24
26
  def self.valid?(source, prism_interpreter: PRISM_INTERPRETER)
25
- return false unless Prism.parse(source).success?
27
+ return false unless Prism.parse_success?(source)
26
28
 
27
29
  prism_interpreter || parse_y_accepts?(source)
28
30
  end
@@ -52,5 +54,5 @@ module Evilution::AST::SyntaxCheck
52
54
  $VERBOSE = verbose
53
55
  end
54
56
 
55
- private_class_method :parse_y_accepts?, :without_warnings
57
+ private_class_method :without_warnings
56
58
  end
@@ -20,16 +20,25 @@ module Evilution::Baseline::Report
20
20
  # failed_ids: every failing example, named so a mutation run can recognise
21
21
  # it again (see Integration::RSpec::ExampleIds); empty where the framework
22
22
  # does not provide such names.
23
- def self.build(passed:, failures: [], error: nil, failed_ids: [])
23
+ #
24
+ # example_durations: the seconds each example took, by the same ids, so a
25
+ # mutation run can try its cheapest examples first.
26
+ def self.build(passed:, failures: [], error: nil, failed_ids: [], example_durations: {})
24
27
  {
25
28
  passed: passed ? true : false,
26
29
  failure_count: failures.length,
27
30
  failures: failures.first(MAX_EXAMPLES).map { |failure| example(failure) },
28
31
  error: error_excerpt(error),
29
- failed_ids: failed_ids.first(MAX_IDS)
32
+ failed_ids: failed_ids.first(MAX_IDS),
33
+ example_durations: timed(example_durations)
30
34
  }
31
35
  end
32
36
 
37
+ def self.timed(durations)
38
+ durations.select { |_, seconds| seconds.is_a?(Numeric) }.first(MAX_IDS).to_h
39
+ end
40
+ private_class_method :timed
41
+
33
42
  # A runner may still answer with a bare pass/fail.
34
43
  def self.from(outcome)
35
44
  outcome.is_a?(Hash) ? outcome : build(passed: outcome)
@@ -3,10 +3,14 @@
3
3
  require_relative "spec_resolver"
4
4
  require_relative "process_cleanup"
5
5
  require_relative "diagnostic"
6
+ require_relative "coverage/load_tracker"
6
7
 
7
8
  class Evilution::Baseline
8
- Result = Struct.new(:failed_spec_files, :duration, :failures) do
9
- def initialize(failed_spec_files:, duration:, failures: [])
9
+ # spec_file_durations: seconds each spec file took, in the order they ran.
10
+ # example_durations: seconds each example took, by the id the integration
11
+ # gives it; empty where the integration does not time its examples.
12
+ Result = Struct.new(:failed_spec_files, :duration, :failures, :spec_file_durations, :example_durations) do
13
+ def initialize(failed_spec_files:, duration:, failures: [], spec_file_durations: {}, example_durations: {})
10
14
  super
11
15
  freeze
12
16
  end
@@ -42,11 +46,19 @@ class Evilution::Baseline
42
46
  end
43
47
 
44
48
  def call(subjects)
45
- start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
46
- failures = baseline_spec_files(subjects).filter_map { |spec_file| check_spec_file(spec_file) }
49
+ start_time = monotonic_now
50
+ durations = {}
51
+ @example_durations = {}
52
+ failures = baseline_spec_files(subjects).filter_map do |spec_file|
53
+ spec_started = monotonic_now
54
+ failure = check_spec_file(spec_file)
55
+ durations[spec_file] = monotonic_now - spec_started
56
+ failure
57
+ end
47
58
 
48
- duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
49
- Result.new(failed_spec_files: failures.to_set(&:spec_file), duration: duration, failures: failures)
59
+ Result.new(failed_spec_files: failures.to_set(&:spec_file), duration: monotonic_now - start_time,
60
+ failures: failures, spec_file_durations: durations.freeze,
61
+ example_durations: @example_durations.freeze)
50
62
  end
51
63
 
52
64
  def run_spec_file(spec_file)
@@ -76,6 +88,8 @@ class Evilution::Baseline
76
88
  runner = @runner
77
89
  Process.fork do
78
90
  read_io.close
91
+ # The spec file may start line coverage itself (see LoadTracker).
92
+ Evilution::Coverage::LoadTracker.release_inherited
79
93
  $stdout.reopen(File::NULL, "w")
80
94
  $stderr.reopen(File::NULL, "w")
81
95
 
@@ -96,6 +110,7 @@ class Evilution::Baseline
96
110
  return warned(SpecFailure.new(spec_file: spec_file, error: unreported_error(status))) if data.empty?
97
111
 
98
112
  report = Marshal.load(data)
113
+ collect_example_durations(report)
99
114
  report[:passed] ? nil : warned(SpecFailure.from_report(spec_file, report))
100
115
  end
101
116
 
@@ -118,6 +133,16 @@ class Evilution::Baseline
118
133
 
119
134
  private
120
135
 
136
+ # Gathered while #call runs, from each spec file's report as it comes in.
137
+ def collect_example_durations(report)
138
+ durations = report[:example_durations]
139
+ @example_durations.merge!(durations) if @example_durations && durations.is_a?(Hash)
140
+ end
141
+
142
+ def monotonic_now
143
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
144
+ end
145
+
121
146
  # Runs in the child. A runner that raises -- a spec helper that cannot load,
122
147
  # a framework that is not there -- is a failure with a reason, not a child
123
148
  # that dies without a word.
@@ -0,0 +1,140 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require_relative "../cache"
5
+ require_relative "../related_spec_heuristic"
6
+ require_relative "skeleton"
7
+ require_relative "spec_digest"
8
+ require_relative "run_fingerprint"
9
+
10
+ # The keys a mutation's verdict is cached under.
11
+ #
12
+ # The subject key stands for everything the verdict of any mutation of one
13
+ # subject depends on that the cache can see: the run's fingerprint (settings,
14
+ # versions, spec helpers), the subject's own source, the rest of its file
15
+ # outside every subject (see Skeleton), and the spec files that resolve for
16
+ # the file. Editing one method changes only that method's key.
17
+ #
18
+ # The mutation key tells the mutations of one subject apart. It is counted
19
+ # from the subject rather than from the top of the file, so a subject that
20
+ # only moved, because lines were added above it, keeps its verdicts.
21
+ #
22
+ # Neither key sees the other methods a subject calls, other source files, or
23
+ # what spec files load beyond the conventional helpers; a change there keeps
24
+ # the cached verdicts. Only method definitions are left out of the skeleton:
25
+ # constants and class-body declarations (callbacks, validations, scopes, AASM
26
+ # machines) stay in it, so editing one changes every key of its file.
27
+ class Evilution::Cache::Keys
28
+ def self.for(config)
29
+ new(
30
+ fingerprint: Evilution::Cache::RunFingerprint.new(config),
31
+ spec_selector: config.spec_selector,
32
+ related_specs: config.related_specs_heuristic? ? Evilution::RelatedSpecHeuristic.new : nil
33
+ )
34
+ end
35
+
36
+ # spec_selector: answers the spec files run for a source file.
37
+ # related_specs: answers the spec files the related-specs heuristic adds
38
+ # for a mutation, when it is on.
39
+ def initialize(fingerprint:, spec_selector:, related_specs: nil, spec_digest: Evilution::Cache::SpecDigest.new)
40
+ @fingerprint = fingerprint
41
+ @spec_selector = spec_selector
42
+ @related_specs = related_specs
43
+ @spec_digest = spec_digest
44
+ @spans = {}
45
+ @names = {}
46
+ @skeletons = {}
47
+ @starts = {}
48
+ @subject_keys = {}.compare_by_identity
49
+ end
50
+
51
+ # Every subject parsed from the run's files, before any is filtered out:
52
+ # the skeleton leaves out every method definition, whichever this run
53
+ # mutates. Everything else stays in it. A constant is read by the methods
54
+ # around it, and a class-body declaration runs at load and decides whether
55
+ # and when methods run (`before_save :normalize, if: :frozen?`), so editing
56
+ # either changes the key of every subject in its file.
57
+ def index(subjects)
58
+ subjects.group_by(&:file_path).each do |file_path, file_subjects|
59
+ @spans[file_path] = file_subjects.select { |subject| subject.kind == :method }.map(&:byte_range)
60
+ @names[file_path] = file_subjects.map(&:name)
61
+ @starts[file_path] = starts(file_subjects)
62
+ end
63
+ end
64
+
65
+ def subject_names(file_path)
66
+ @names[file_path]
67
+ end
68
+
69
+ # nil when the mutation cannot be keyed: its sources were released, or no
70
+ # spec file resolves for its file.
71
+ def subject_key(mutation)
72
+ source = mutation.original_source
73
+ return nil if source.nil?
74
+
75
+ subject = mutation.subject
76
+ return @subject_keys[subject] if @subject_keys.key?(subject)
77
+
78
+ @subject_keys[subject] = build_subject_key(mutation, subject, source)
79
+ end
80
+
81
+ def mutation_key(mutation)
82
+ return nil if mutation.mutated_source.nil?
83
+
84
+ first_line = mutation.changed_lines.first - mutation.subject.line_number
85
+ key = [mutation.operator_name, first_line, edit_digest(mutation)].join(":")
86
+ related = related_digest(mutation)
87
+ related ? "#{key}:#{related}" : key
88
+ end
89
+
90
+ private
91
+
92
+ def starts(subjects)
93
+ subjects.filter_map { |subject| [subject.name, subject.byte_range.begin] if subject.byte_range }
94
+ end
95
+
96
+ def build_subject_key(mutation, subject, source)
97
+ specs = @spec_digest.call(@spec_selector.call(mutation.file_path))
98
+ return nil unless specs
99
+
100
+ Digest::SHA256.hexdigest(
101
+ [@fingerprint.to_s, mutation.file_path, subject.name, place(subject), skeleton(mutation.file_path, source),
102
+ Digest::SHA256.hexdigest(subject.source), specs].join("\0")
103
+ )
104
+ end
105
+
106
+ # Tells apart subjects of one file that share a name and a source, such as
107
+ # a method defined twice: which of the same-named subjects it is, counted
108
+ # from the top of the file, so that it keeps its key when it only moves.
109
+ # Without that count -- the file was not indexed, or the subject has no
110
+ # byte range -- the line it starts on, so such a subject loses its
111
+ # verdicts when it moves.
112
+ def place(subject)
113
+ starts = @starts[subject.file_path]
114
+ range = subject.byte_range
115
+ return "@#{subject.line_number}" if starts.nil? || range.nil?
116
+
117
+ starts.count { |name, start| name == subject.name && start < range.begin }.to_s
118
+ end
119
+
120
+ def skeleton(file_path, source)
121
+ @skeletons[file_path] ||= Evilution::Cache::Skeleton.digest(source, @spans.fetch(file_path, []))
122
+ end
123
+
124
+ # The affected lines before and after, which with the line the edit starts
125
+ # on fix the mutated file. A mutation built without them falls back to the
126
+ # whole mutated file.
127
+ def edit_digest(mutation)
128
+ original = mutation.original_slice
129
+ mutated = mutation.mutated_slice
130
+ text = original && mutated ? "#{original}\0#{mutated}" : mutation.mutated_source
131
+ Digest::SHA256.hexdigest(text)[0, 16]
132
+ end
133
+
134
+ def related_digest(mutation)
135
+ return nil unless @related_specs
136
+
137
+ digest = @spec_digest.call(@related_specs.call(mutation))
138
+ digest && digest[0, 16]
139
+ end
140
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "json"
5
+ require_relative "../cache"
6
+ require_relative "../version"
7
+ require_relative "spec_digest"
8
+
9
+ # What a cached verdict depends on beyond the subject and its spec files: the
10
+ # settings that decide how a mutation is run and judged, the evilution and
11
+ # Ruby versions, the locked gems, and the helpers every spec file loads.
12
+ # Settings that only shape the output or the scheduling (format, jobs,
13
+ # progress) are left out, so changing them keeps the cache.
14
+ class Evilution::Cache::RunFingerprint
15
+ SETTINGS = %i[
16
+ integration isolation profile timeout adaptive_timeout? timeout_factor timeout_floor timeout_ceiling
17
+ baseline example_targeting example_targeting_fallback example_targeting_strategy
18
+ related_specs_heuristic fallback_to_full_suite preload stop_on_kill killer_first? warmup skip_uncovered?
19
+ probe_first? copy_worker_databases?
20
+ ].freeze
21
+
22
+ HELPERS = %w[
23
+ .rspec spec/spec_helper.rb spec/rails_helper.rb spec/support test/test_helper.rb test/support
24
+ Gemfile.lock gems.locked
25
+ ].freeze
26
+
27
+ def initialize(config, spec_digest: Evilution::Cache::SpecDigest.new)
28
+ @config = config
29
+ @spec_digest = spec_digest
30
+ end
31
+
32
+ def to_s
33
+ @to_s ||= Digest::SHA256.hexdigest(JSON.generate(payload))
34
+ end
35
+
36
+ private
37
+
38
+ def payload
39
+ {
40
+ "evilution" => Evilution::VERSION,
41
+ "ruby" => "#{RUBY_ENGINE} #{RUBY_VERSION}",
42
+ "settings" => SETTINGS.to_h { |name| [name.to_s, @config.public_send(name).to_s] },
43
+ "helpers" => @spec_digest.call(helper_paths).to_s
44
+ }
45
+ end
46
+
47
+ def helper_paths
48
+ preload = @config.preload
49
+ preload ? HELPERS + [preload.to_s] : HELPERS
50
+ end
51
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require_relative "../cache"
5
+
6
+ # A digest of what a source file holds outside the spans it is given (its
7
+ # method definitions, see Cache::Keys#index): the class and module lines,
8
+ # constants, `include`s, `attr_*`s, declarations and other class-body code.
9
+ # Editing a method leaves it unchanged; editing anything the spans do not
10
+ # cover changes it.
11
+ #
12
+ # Each removed span is marked where it was, so text cannot move from one
13
+ # side of a subject to the other unnoticed. Without a span for every subject
14
+ # the whole file is digested, which is what the cache keyed on before
15
+ # subjects had keys of their own.
16
+ module Evilution::Cache::Skeleton
17
+ SEPARATOR = "\0"
18
+ private_constant :SEPARATOR
19
+
20
+ def self.digest(source, spans)
21
+ bytes = source.b
22
+ merged = merge(spans, bytes.bytesize)
23
+ return Digest::SHA256.hexdigest(bytes) unless merged
24
+
25
+ Digest::SHA256.hexdigest(outside(bytes, merged))
26
+ end
27
+
28
+ # The spans in order with overlapping and nested ones folded together, or
29
+ # nil when one is unknown or does not fit the file.
30
+ def self.merge(spans, size)
31
+ return nil unless spans.all? { |span| span.is_a?(Range) && span.end <= size }
32
+
33
+ spans.sort_by(&:begin).each_with_object([]) { |span, merged| fold(merged, span) }
34
+ end
35
+ private_class_method :merge
36
+
37
+ def self.fold(merged, span)
38
+ last = merged.last
39
+ if last && span.begin < last.end
40
+ merged[-1] = last.begin...[last.end, span.end].max
41
+ else
42
+ merged << (span.begin...span.end)
43
+ end
44
+ end
45
+ private_class_method :fold
46
+
47
+ def self.outside(bytes, spans)
48
+ kept = +""
49
+ cursor = 0
50
+ spans.each do |span|
51
+ kept << bytes.byteslice(cursor, span.begin - cursor) << SEPARATOR
52
+ cursor = span.end
53
+ end
54
+ kept << bytes.byteslice(cursor, bytes.bytesize - cursor)
55
+ end
56
+ private_class_method :outside
57
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require_relative "../cache"
5
+
6
+ # A digest of the contents of a set of test files. A directory stands for the
7
+ # Ruby files under it, and a path with no file behind it digests as missing,
8
+ # so a file appearing or going away changes the digest too.
9
+ #
10
+ # Each file is read once for the life of the instance, which belongs to one
11
+ # run: the same spec files resolve for every subject of a source file.
12
+ class Evilution::Cache::SpecDigest
13
+ MISSING = "missing"
14
+ private_constant :MISSING
15
+
16
+ def initialize
17
+ @files = {}
18
+ end
19
+
20
+ # nil when there are no paths: there is nothing to vouch for the verdict.
21
+ def call(paths)
22
+ expanded = Array(paths).flat_map { |path| expand(path.to_s) }.uniq.sort
23
+ return nil if expanded.empty?
24
+
25
+ Digest::SHA256.hexdigest(expanded.map { |path| "#{path}\0#{file_digest(path)}" }.join("\n"))
26
+ end
27
+
28
+ private
29
+
30
+ def expand(path)
31
+ return [path] unless File.directory?(path)
32
+
33
+ Dir.glob(File.join(path, "**", "*.rb"))
34
+ end
35
+
36
+ def file_digest(path)
37
+ @files[path] ||= File.file?(path) ? Digest::SHA256.hexdigest(File.binread(path)) : MISSING
38
+ end
39
+ end