evilution 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +62 -0
  3. data/.rubocop_todo.yml +5 -0
  4. data/CHANGELOG.md +77 -0
  5. data/README.md +80 -24
  6. data/docs/architecture.md +89 -9
  7. data/docs/ast_pattern_syntax.md +50 -5
  8. data/docs/isolation.md +39 -2
  9. data/docs/migration-from-mutant.md +1 -1
  10. data/lib/evilution/ast/aasm_declaration.rb +133 -0
  11. data/lib/evilution/ast/callback_declaration.rb +91 -0
  12. data/lib/evilution/ast/included_block.rb +27 -0
  13. data/lib/evilution/ast/literal_callable.rb +21 -0
  14. data/lib/evilution/ast/parser.rb +119 -16
  15. data/lib/evilution/ast/pattern/method_name.rb +63 -0
  16. data/lib/evilution/ast/pattern/parser.rb +14 -15
  17. data/lib/evilution/ast/regexp_pattern.rb +104 -0
  18. data/lib/evilution/ast/scope_declaration.rb +45 -0
  19. data/lib/evilution/ast/uncovered_code.rb +88 -0
  20. data/lib/evilution/ast/value_object_definition.rb +25 -0
  21. data/lib/evilution/baseline/failure_formatter.rb +27 -0
  22. data/lib/evilution/baseline/report.rb +66 -0
  23. data/lib/evilution/baseline/spec_failure.rb +38 -0
  24. data/lib/evilution/baseline.rb +67 -30
  25. data/lib/evilution/cli/parser/file_args.rb +2 -1
  26. data/lib/evilution/cli/parser/options_builder.rb +1 -1
  27. data/lib/evilution/cli.rb +3 -2
  28. data/lib/evilution/config/validators/spec_mappings.rb +2 -1
  29. data/lib/evilution/config.rb +3 -2
  30. data/lib/evilution/equivalent/detector.rb +3 -1
  31. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/condition.rb +66 -0
  32. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/disturbance.rb +54 -0
  33. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/early_exit.rb +73 -0
  34. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/guard.rb +36 -0
  35. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/index_read.rb +82 -0
  36. data/lib/evilution/equivalent/heuristic/guarded_index_fetch/node_path.rb +43 -0
  37. data/lib/evilution/equivalent/heuristic/guarded_index_fetch.rb +65 -0
  38. data/lib/evilution/example_filter.rb +45 -0
  39. data/lib/evilution/hooks/registry.rb +2 -1
  40. data/lib/evilution/integration/base.rb +2 -0
  41. data/lib/evilution/integration/known_failures.rb +26 -0
  42. data/lib/evilution/integration/loading/body_call_neutralizer.rb +132 -13
  43. data/lib/evilution/integration/loading/callback_redeclaration.rb +186 -0
  44. data/lib/evilution/integration/loading/concern_redeclaration.rb +61 -0
  45. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -12
  46. data/lib/evilution/integration/loading/mutation_applier.rb +16 -0
  47. data/lib/evilution/integration/loading/test_class_cache.rb +46 -0
  48. data/lib/evilution/integration/minitest/test_ids.rb +21 -0
  49. data/lib/evilution/integration/minitest.rb +45 -6
  50. data/lib/evilution/integration/rspec/baseline_runner.rb +47 -3
  51. data/lib/evilution/integration/rspec/example_ids.rb +35 -0
  52. data/lib/evilution/integration/rspec/result_builder.rb +6 -1
  53. data/lib/evilution/integration/rspec/state_guard/anonymous_example_group_examples.rb +36 -0
  54. data/lib/evilution/integration/rspec/state_guard.rb +4 -1
  55. data/lib/evilution/integration/rspec.rb +22 -4
  56. data/lib/evilution/integration/test_unit/result_builder.rb +6 -0
  57. data/lib/evilution/integration/test_unit/test_ids.rb +18 -0
  58. data/lib/evilution/integration/test_unit.rb +37 -9
  59. data/lib/evilution/isolation/fork.rb +2 -1
  60. data/lib/evilution/isolation/in_process.rb +21 -4
  61. data/lib/evilution/mcp/info_tool/status_glossary.rb +2 -2
  62. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +2 -1
  63. data/lib/evilution/memory/leak_check.rb +27 -3
  64. data/lib/evilution/mutation.rb +7 -2
  65. data/lib/evilution/mutator/base.rb +49 -5
  66. data/lib/evilution/mutator/operator/alias_removal.rb +126 -0
  67. data/lib/evilution/mutator/operator/argument_order_permutation.rb +101 -0
  68. data/lib/evilution/mutator/operator/argument_removal.rb +9 -1
  69. data/lib/evilution/mutator/operator/comparison_operand_swap.rb +54 -0
  70. data/lib/evilution/mutator/operator/data_struct_member.rb +82 -0
  71. data/lib/evilution/mutator/operator/exception_swallow.rb +84 -0
  72. data/lib/evilution/mutator/operator/format_specifier_swap.rb +100 -0
  73. data/lib/evilution/mutator/operator/forwarded_argument_drop.rb +142 -0
  74. data/lib/evilution/mutator/operator/integer_division_to_fdiv.rb +62 -0
  75. data/lib/evilution/mutator/operator/keyword_argument.rb +25 -2
  76. data/lib/evilution/mutator/operator/keyword_value_swap.rb +75 -0
  77. data/lib/evilution/mutator/operator/no_matching_pattern_else.rb +47 -0
  78. data/lib/evilution/mutator/operator/numbered_parameter_swap.rb +78 -0
  79. data/lib/evilution/mutator/operator/off_by_one_boundary.rb +68 -0
  80. data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +14 -3
  81. data/lib/evilution/mutator/operator/pattern_matching_array.rb +12 -1
  82. data/lib/evilution/mutator/operator/pattern_wildcard_widening.rb +117 -0
  83. data/lib/evilution/mutator/operator/pin_operator_removal.rb +48 -0
  84. data/lib/evilution/mutator/operator/regex_simplification.rb +53 -132
  85. data/lib/evilution/mutator/operator/regexp_alternation_branch_deletion.rb +54 -0
  86. data/lib/evilution/mutator/operator/regexp_anchor_promotion.rb +43 -0
  87. data/lib/evilution/mutator/operator/regexp_capture_to_passive.rb +61 -0
  88. data/lib/evilution/mutator/operator/regexp_character_type_complement.rb +54 -0
  89. data/lib/evilution/mutator/operator/regexp_named_group_rename.rb +106 -0
  90. data/lib/evilution/mutator/operator/regexp_option_removal.rb +51 -0
  91. data/lib/evilution/mutator/operator/regexp_quantifier_minimum_swap.rb +45 -0
  92. data/lib/evilution/mutator/operator/rescue_else_concatenation.rb +62 -0
  93. data/lib/evilution/mutator/operator/rescue_handler_concatenation.rb +69 -0
  94. data/lib/evilution/mutator/operator/rescue_handler_promotion.rb +65 -0
  95. data/lib/evilution/mutator/operator/return_keyword_removal.rb +79 -0
  96. data/lib/evilution/mutator/operator/rightward_assignment.rb +46 -0
  97. data/lib/evilution/mutator/operator/send_mutation.rb +2 -0
  98. data/lib/evilution/mutator/operator/splat_operator.rb +38 -13
  99. data/lib/evilution/mutator/operator/statement_reorder.rb +153 -0
  100. data/lib/evilution/mutator/registry.rb +31 -2
  101. data/lib/evilution/mutator/rescue_handlers.rb +81 -0
  102. data/lib/evilution/process_supervisor.rb +3 -2
  103. data/lib/evilution/reporter/cli/line_formatters/baseline_neutralized_notice.rb +55 -0
  104. data/lib/evilution/reporter/cli/metrics_block.rb +2 -0
  105. data/lib/evilution/reporter/json/baseline.rb +15 -0
  106. data/lib/evilution/reporter/json.rb +7 -2
  107. data/lib/evilution/result/baseline_neutralization.rb +10 -0
  108. data/lib/evilution/result/mutation_result.rb +9 -1
  109. data/lib/evilution/result/summary.rb +42 -2
  110. data/lib/evilution/runner/baseline_runner.rb +9 -4
  111. data/lib/evilution/runner/canary.rb +2 -57
  112. data/lib/evilution/runner/canary_failure_message.rb +101 -0
  113. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +22 -8
  114. data/lib/evilution/runner/mutation_executor/result_cache.rb +3 -0
  115. data/lib/evilution/runner/mutation_executor/result_packer.rb +4 -2
  116. data/lib/evilution/runner/mutation_executor.rb +1 -1
  117. data/lib/evilution/runner/mutation_planner.rb +11 -2
  118. data/lib/evilution/runner/report_publisher.rb +3 -2
  119. data/lib/evilution/runner/subject_pipeline.rb +46 -1
  120. data/lib/evilution/runner.rb +4 -1
  121. data/lib/evilution/subject.rb +8 -2
  122. data/lib/evilution/version.rb +1 -1
  123. data/lib/evilution.rb +28 -0
  124. data/script/memory_check +62 -16
  125. metadata +83 -4
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../runner"
4
+
5
+ # Explains why the proof-of-life canary failed, from the result its synthetic
6
+ # mutation came back with. Kept apart from Canary, which only runs the probe:
7
+ # the wording changes for different reasons than the mechanics do.
8
+ class Evilution::Runner::CanaryFailureMessage
9
+ # Raised by RubyGems when it cannot load a gem at a version that satisfies
10
+ # every requirement already in play.
11
+ GEM_ACTIVATION_ERRORS = %w[
12
+ Gem::ConflictError Gem::LoadError Gem::MissingSpecError Gem::MissingSpecVersionError
13
+ ].freeze
14
+
15
+ def initialize(result)
16
+ @result = result
17
+ end
18
+
19
+ # When the child reported an error, that error IS the diagnosis -- naming it
20
+ # beats guessing. The speculative list stays only for the cases that carry no
21
+ # error at all (:killed, :timeout), where guesses are the only help there is.
22
+ # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
23
+ # this message used to drop, and none of the four guesses was the cause.
24
+ # EV-65nf / GH #1586.
25
+ def to_s
26
+ return gem_activation_message(@result) if gem_activation_failure?(@result)
27
+
28
+ "#{failure_preamble(@result.status)} #{diagnosis(@result)} " \
29
+ "Re-run with --no-canary to bypass this check."
30
+ end
31
+
32
+ private
33
+
34
+ # The usual cause is evilution running outside `bundle exec`: RubyGems then
35
+ # activates the newest installed copy of a shared dependency, which the test
36
+ # framework may not accept. Every mutation would fail the same way, so
37
+ # skipping the canary would not help.
38
+ def gem_activation_message(result)
39
+ "evilution proof-of-life canary failed: the test framework could not be loaded " \
40
+ "because of a gem activation conflict. This is an environment problem, not a " \
41
+ "mutation-pipeline defect, and every mutation would fail the same way. Run " \
42
+ "evilution under `bundle exec` so the project's Gemfile.lock decides gem versions " \
43
+ "(for the MCP server: command `bundle`, args [\"exec\", \"evilution\", \"mcp\"]). " \
44
+ "The child reported: #{reported_error(result)}"
45
+ end
46
+
47
+ # The class may arrive in its own field or only as the message prefix,
48
+ # depending on which path packed the error.
49
+ def gem_activation_failure?(result)
50
+ message = result.error_message.to_s
51
+
52
+ GEM_ACTIVATION_ERRORS.any? do |klass|
53
+ result.error_class == klass || message.start_with?("#{klass}:")
54
+ end
55
+ end
56
+
57
+ def failure_preamble(status)
58
+ "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
59
+ "mutation was scored #{status.inspect} instead of :survived. The mutation " \
60
+ "pipeline is misreporting — every score this run would produce is unreliable."
61
+ end
62
+
63
+ def diagnosis(result)
64
+ reported = reported_error(result)
65
+ return speculative_causes if reported.nil?
66
+
67
+ "The child reported: #{reported}"
68
+ end
69
+
70
+ def reported_error(result)
71
+ message = result.error_message
72
+ return nil if message.nil? || message.empty?
73
+
74
+ described = with_class_prefix(message, result.error_class)
75
+ frame = first_frame(result)
76
+ frame.nil? ? described : "#{described} (at #{frame})"
77
+ end
78
+
79
+ # MutationApplier already prefixes the class onto the message it packs
80
+ # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
81
+ # leave the class in its own field. Prefixing unconditionally would print
82
+ # "NameError: NameError: ..." for the former.
83
+ def with_class_prefix(message, klass)
84
+ return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
85
+
86
+ "#{klass}: #{message}"
87
+ end
88
+
89
+ def first_frame(result)
90
+ backtrace = result.error_backtrace
91
+ return nil if backtrace.nil? || backtrace.empty?
92
+
93
+ backtrace.first
94
+ end
95
+
96
+ def speculative_causes
97
+ "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
98
+ "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
99
+ "an isolation-mode defect."
100
+ end
101
+ end
@@ -4,6 +4,17 @@ require_relative "../neutralizer"
4
4
  require_relative "../../../result/mutation_result"
5
5
  require_relative "../../../result/neutral_reason"
6
6
 
7
+ # Reclassifies a kill as :neutral when the tests failed on nothing but
8
+ # examples that were already failing before any mutation ran.
9
+ #
10
+ # Such an example fails whatever the mutation is, so every mutation it runs
11
+ # against would count as killed and a red spec file would score full marks.
12
+ # The failure says nothing about the mutation: it has no verdict.
13
+ #
14
+ # A survivor is left alone, whatever the baseline did. Its tests passed, the
15
+ # examples that were red in the baseline included, so the gap it reports is
16
+ # real -- and neutralizing it, as used to happen for every survivor a red spec
17
+ # file covered, hid exactly the mutations a reader needs to see.
7
18
  class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
8
19
  def initialize(config:, spec_resolver:, fallback_dir:)
9
20
  @config = config
@@ -12,19 +23,22 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
12
23
  end
13
24
 
14
25
  def call(result, baseline_result:)
15
- return result unless result.survived? && baseline_result && baseline_result.failed?
26
+ return result unless result.killed? && result.known_failures_only?
16
27
 
17
- return neutralize(result, nil) if @config.spec_files.any?
18
-
19
- failed = baseline_result.failed_spec_files
20
- spec_file = covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
21
- return result unless spec_file
22
-
23
- neutralize(result, spec_file)
28
+ neutralize(result, red_spec(result, baseline_result))
24
29
  end
25
30
 
26
31
  private
27
32
 
33
+ # The red spec file to name: one covering the mutated source. A run given
34
+ # its spec files explicitly has no single one to point at.
35
+ def red_spec(result, baseline_result)
36
+ return nil if @config.spec_files.any? || baseline_result.nil?
37
+
38
+ failed = baseline_result.failed_spec_files
39
+ covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
40
+ end
41
+
28
42
  # The spec files the baseline ran for this source: every file the resolver
29
43
  # returns (a spec_selector may map one source to several), or the fallback
30
44
  # directory when nothing resolves -- but only if the run falls back to the
@@ -34,6 +34,9 @@ class Evilution::Runner::MutationExecutor::ResultCache
34
34
  return unless @backend
35
35
  return unless result.killed? || result.timeout?
36
36
  return if infra_crash?(result)
37
+ # Nor is one where only already-failing examples failed: the baseline
38
+ # that says so belongs to this run, and the cache keeps no trace of it.
39
+ return if result.known_failures_only?
37
40
 
38
41
  @backend.store(mutation,
39
42
  status: result.status,
@@ -15,7 +15,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
15
15
  parent_rss_kb: result.parent_rss_kb,
16
16
  error_message: result.error_message,
17
17
  error_class: result.error_class,
18
- error_backtrace: result.error_backtrace
18
+ error_backtrace: result.error_backtrace,
19
+ known_failures_only: result.known_failures_only?
19
20
  }
20
21
  end
21
22
 
@@ -35,7 +36,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
35
36
  message: data[:error_message],
36
37
  klass: data[:error_class],
37
38
  backtrace: data[:error_backtrace]
38
- )
39
+ ),
40
+ known_failures_only: data[:known_failures_only]
39
41
  )
40
42
  end
41
43
  end
@@ -30,7 +30,7 @@ class Evilution::Runner::MutationExecutor
30
30
  end
31
31
 
32
32
  def call(mutations, baseline_result = nil)
33
- integration = @baseline_runner.build_integration
33
+ integration = @baseline_runner.build_integration(baseline_result)
34
34
  spec_resolver = baseline_failed?(baseline_result) ? @baseline_runner.neutralization_resolver : nil
35
35
  notifier = build_notifier
36
36
  pipeline = build_pipeline(spec_resolver)
@@ -7,7 +7,7 @@ require_relative "../ast/pattern/filter"
7
7
  require_relative "../equivalent/detector"
8
8
 
9
9
  class Evilution::Runner::MutationPlanner
10
- Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, keyword_init: true)
10
+ Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, :mutated_lines, keyword_init: true)
11
11
 
12
12
  GenerationResult = Data.define(:mutations, :skipped)
13
13
  DisabledFilterResult = Data.define(:enabled, :disabled)
@@ -33,13 +33,22 @@ class Evilution::Runner::MutationPlanner
33
33
  sig_filter = filter_sig_blocks(disabled_filter.enabled)
34
34
  equivalent_filter = filter_equivalent(sig_filter.enabled)
35
35
 
36
- build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
36
+ plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
37
+ plan.mutated_lines = mutated_lines(deduped)
38
+ plan
37
39
  end
38
40
 
39
41
  private
40
42
 
41
43
  attr_reader :config, :registry
42
44
 
45
+ # The lines mutations were generated for, per file. Taken before any filter:
46
+ # a line whose mutations were all disabled or judged equivalent was still
47
+ # reached by an operator.
48
+ def mutated_lines(mutations)
49
+ mutations.group_by(&:file_path).transform_values { |group| group.map(&:line).uniq.sort }
50
+ end
51
+
43
52
  def compute_disabled_mutations(disabled_filter)
44
53
  return [] unless config.show_disabled?
45
54
 
@@ -5,6 +5,7 @@ require_relative "../reporter/json"
5
5
  require_relative "../reporter/cli"
6
6
  require_relative "../reporter/html"
7
7
  require_relative "../session/store"
8
+ require_relative "../diagnostic"
8
9
 
9
10
  class Evilution::Runner::ReportPublisher
10
11
  def initialize(config)
@@ -46,7 +47,7 @@ class Evilution::Runner::ReportPublisher
46
47
  def publish_html(output)
47
48
  path = "evilution-report.html"
48
49
  File.write(path, output)
49
- warn "HTML report written to #{path}"
50
+ Evilution::Diagnostic.warn("HTML report written to #{path}")
50
51
  end
51
52
 
52
53
  def save_session(summary)
@@ -54,7 +55,7 @@ class Evilution::Runner::ReportPublisher
54
55
 
55
56
  Evilution::Session::Store.new.save(summary)
56
57
  rescue StandardError => e
57
- warn "[evilution] failed to save session: #{e.message}" unless config.quiet
58
+ Evilution::Diagnostic.warn("[evilution] failed to save session: #{e.message}") unless config.quiet
58
59
  end
59
60
 
60
61
  private
@@ -2,6 +2,8 @@
2
2
 
3
3
  require_relative "../runner"
4
4
  require_relative "../ast/inheritance_scanner"
5
+ require_relative "../ast/uncovered_code"
6
+ require_relative "../diagnostic"
5
7
  require_relative "../git/changed_files"
6
8
 
7
9
  class Evilution::Runner::SubjectPipeline
@@ -12,6 +14,7 @@ class Evilution::Runner::SubjectPipeline
12
14
 
13
15
  def call
14
16
  subjects = parse_subjects
17
+ @subjects_by_file = subjects.group_by(&:file_path)
15
18
  subjects = filter_by_descendants(subjects) if descendants_target?
16
19
  subjects = filter_by_target(subjects) if method_target?
17
20
  subjects = filter_by_line_ranges(subjects) if config.line_ranges?
@@ -22,6 +25,28 @@ class Evilution::Runner::SubjectPipeline
22
25
  @target_files ||= resolve_target_files
23
26
  end
24
27
 
28
+ # Warns about the targeted lines that hold code outside every subject and
29
+ # that no mutation reached either. mutated_lines: the lines mutations were
30
+ # generated for, per file -- known only once #call's subjects are planned.
31
+ #
32
+ # A line range is checked as given; a whole file only when it has no
33
+ # subjects at all -- every class with an `include` or a constant would
34
+ # otherwise warn.
35
+ def report_uncovered_code(mutated_lines = {})
36
+ by_file = @subjects_by_file || {}
37
+ @subjects_by_file = nil
38
+ @uncovered_code = target_files.filter_map do |file|
39
+ uncovered_entry(file, by_file.fetch(file, []), mutated_lines.fetch(file, []))
40
+ end
41
+ @uncovered_code.each { |entry| warn_uncovered(entry) }
42
+ end
43
+
44
+ # Those lines, as `{ file:, lines: ["2-4", "9"] }` per file, once
45
+ # #report_uncovered_code has run.
46
+ def uncovered_code
47
+ @uncovered_code || []
48
+ end
49
+
25
50
  private
26
51
 
27
52
  attr_reader :config, :parser
@@ -30,6 +55,26 @@ class Evilution::Runner::SubjectPipeline
30
55
  target_files.flat_map { |file| parser.call(file) }
31
56
  end
32
57
 
58
+ def uncovered_entry(file, file_subjects, mutated_lines)
59
+ range = config.line_ranges[file]
60
+ return unless range || file_subjects.empty?
61
+
62
+ lines = Evilution::AST::UncoveredCode.call(file, file_subjects, lines: range, mutated_lines: mutated_lines)
63
+ { file: file, lines: lines.map { |run| line_label(run) } } unless lines.empty?
64
+ end
65
+
66
+ # Spelled the way a range is passed on the command line: `2-4`, or `9`.
67
+ def line_label(run)
68
+ run.size == 1 ? run.first.to_s : "#{run.first}-#{run.last}"
69
+ end
70
+
71
+ def warn_uncovered(entry)
72
+ Evilution::Diagnostic.warn(
73
+ "[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject " \
74
+ "(class-body code such as DSL calls and constants is not mutated); no mutations target those lines."
75
+ )
76
+ end
77
+
33
78
  def source_glob_target?
34
79
  config.target&.start_with?("source:")
35
80
  end
@@ -124,7 +169,7 @@ class Evilution::Runner::SubjectPipeline
124
169
  end
125
170
 
126
171
  def class_matcher(target)
127
- ->(s) { s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
172
+ ->(s) { s.name == target || s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
128
173
  end
129
174
 
130
175
  def filter_by_line_ranges(subjects)
@@ -38,6 +38,7 @@ class Evilution::Runner
38
38
  baseline_result = run_baseline(subjects)
39
39
 
40
40
  plan = mutation_planner.call(subjects)
41
+ subject_pipeline.report_uncovered_code(plan.mutated_lines)
41
42
  release_subject_nodes(subjects)
42
43
  clear_operator_caches
43
44
  execution = run_mutations(plan.enabled, baseline_result)
@@ -52,7 +53,9 @@ class Evilution::Runner
52
53
  disabled_mutations: plan.disabled_mutations,
53
54
  unresolved_target_files: target_spec_audit.call(audited_files),
54
55
  target_file_count: audited_files.length,
55
- infra_retried: execution.infra_retried)
56
+ infra_retried: execution.infra_retried,
57
+ uncovered_code: subject_pipeline.uncovered_code,
58
+ baseline_failures: baseline_result ? baseline_result.failures : [])
56
59
  output_report(summary)
57
60
  save_session(summary)
58
61
 
@@ -1,14 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # What mutations are generated for and reported against. A :method subject is
4
+ # a method definition; a :constant subject is a definition assigned to a
5
+ # constant outside any method, such as `Point = Data.define(:x, :y)`. A :scope
6
+ # or :aasm subject is the body of a callable handed to a class-body
7
+ # declaration.
3
8
  class Evilution::Subject
4
- attr_reader :name, :file_path, :line_number, :source, :node
9
+ attr_reader :name, :file_path, :line_number, :source, :node, :kind
5
10
 
6
- def initialize(name:, file_path:, line_number:, source:, node:)
11
+ def initialize(name:, file_path:, line_number:, source:, node:, kind: :method)
7
12
  @name = name
8
13
  @file_path = file_path
9
14
  @line_number = line_number
10
15
  @source = source
11
16
  @node = node
17
+ @kind = kind
12
18
  end
13
19
 
14
20
  def release_node!
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Evilution
4
- VERSION = "1.3.0"
4
+ VERSION = "1.4.0"
5
5
  end
data/lib/evilution.rb CHANGED
@@ -14,6 +14,7 @@ require_relative "evilution/ast/inheritance_scanner"
14
14
  require_relative "evilution/ast/pattern"
15
15
  require_relative "evilution/ast/sorbet_sig_detector"
16
16
  require_relative "evilution/ast/local_reads"
17
+ require_relative "evilution/ast/regexp_pattern"
17
18
  require_relative "evilution/ast/pattern/matcher"
18
19
  require_relative "evilution/ast/pattern/parser"
19
20
  require_relative "evilution/hooks"
@@ -53,6 +54,33 @@ require_relative "evilution/mutator/operator/receiver_constructor_swap"
53
54
  require_relative "evilution/mutator/operator/block_body_to_nil"
54
55
  require_relative "evilution/mutator/operator/block_body_to_raise"
55
56
  require_relative "evilution/mutator/operator/block_body_promotion"
57
+ require_relative "evilution/mutator/operator/data_struct_member"
58
+ require_relative "evilution/mutator/operator/pin_operator_removal"
59
+ require_relative "evilution/mutator/operator/rightward_assignment"
60
+ require_relative "evilution/mutator/operator/numbered_parameter_swap"
61
+ require_relative "evilution/mutator/operator/forwarded_argument_drop"
62
+ require_relative "evilution/mutator/operator/pattern_wildcard_widening"
63
+ require_relative "evilution/mutator/operator/no_matching_pattern_else"
64
+ require_relative "evilution/mutator/operator/argument_order_permutation"
65
+ require_relative "evilution/mutator/operator/keyword_value_swap"
66
+ require_relative "evilution/mutator/operator/comparison_operand_swap"
67
+ require_relative "evilution/mutator/operator/integer_division_to_fdiv"
68
+ require_relative "evilution/mutator/operator/off_by_one_boundary"
69
+ require_relative "evilution/mutator/operator/format_specifier_swap"
70
+ require_relative "evilution/mutator/operator/exception_swallow"
71
+ require_relative "evilution/mutator/operator/statement_reorder"
72
+ require_relative "evilution/mutator/operator/alias_removal"
73
+ require_relative "evilution/mutator/operator/regexp_character_type_complement"
74
+ require_relative "evilution/mutator/operator/regexp_anchor_promotion"
75
+ require_relative "evilution/mutator/operator/regexp_alternation_branch_deletion"
76
+ require_relative "evilution/mutator/operator/regexp_quantifier_minimum_swap"
77
+ require_relative "evilution/mutator/operator/regexp_capture_to_passive"
78
+ require_relative "evilution/mutator/operator/regexp_named_group_rename"
79
+ require_relative "evilution/mutator/operator/regexp_option_removal"
80
+ require_relative "evilution/mutator/operator/rescue_handler_promotion"
81
+ require_relative "evilution/mutator/operator/rescue_handler_concatenation"
82
+ require_relative "evilution/mutator/operator/rescue_else_concatenation"
83
+ require_relative "evilution/mutator/operator/return_keyword_removal"
56
84
  require_relative "evilution/mutator/operator/arithmetic_replacement"
57
85
  require_relative "evilution/mutator/operator/string_literal"
58
86
  require_relative "evilution/mutator/operator/array_literal"
data/script/memory_check CHANGED
@@ -11,6 +11,13 @@ COMPLEX_FIXTURE = File.expand_path("../spec/support/fixtures/memory_check/target
11
11
  COMPLEX_FIXTURE_SPEC = File.expand_path("../spec/support/fixtures/memory_check/target_examples.rb", __dir__)
12
12
  ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_ITERATIONS", 50))
13
13
  MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_MAX_GROWTH_KB", 10_240))
14
+ # Every check that picks mutations at random draws from its own Random seeded
15
+ # with this, so a run exercises the same mutations every time and a failure can
16
+ # be replayed by passing the printed seed back.
17
+ SEED = Integer(ENV.fetch("MEMORY_CHECK_SEED", 42))
18
+ COMPLEX_POOL_SIZE = 10
19
+ COMPLEX_ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_ITERATIONS", 100))
20
+ COMPLEX_MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_MAX_GROWTH_KB", 10_240))
14
21
 
15
22
  def setup_workload
16
23
  parser = Evilution::AST::Parser.new
@@ -21,25 +28,40 @@ def setup_workload
21
28
  mutations
22
29
  end
23
30
 
24
- def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, &)
25
- check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb)
31
+ def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, picked: nil,
32
+ warmup_iterations: Evilution::Memory::LeakCheck::WARMUP_ITERATIONS, &)
33
+ check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb,
34
+ warmup_iterations: warmup_iterations)
26
35
  result = check.run(&)
27
- report(name, result)
36
+ report(name, result, picked)
28
37
  result[:passed]
29
38
  end
30
39
 
31
- def report(name, result)
40
+ def report(name, result, picked)
32
41
  status = result[:passed] ? "PASS" : "FAIL"
33
42
  metrics = format_metrics(result)
34
43
 
35
44
  puts "[#{status}] #{name}"
36
- puts " Growth: #{metrics[:growth]} (max: #{metrics[:max]})"
45
+ puts " Sustained growth: #{metrics[:sustained]} (max: #{metrics[:max]})"
46
+ puts " Growth: #{metrics[:growth]}"
37
47
  puts " Samples (MB): #{metrics[:samples]}"
48
+ report_picked(picked) unless result[:passed]
38
49
  puts
39
50
  end
40
51
 
52
+ # What a failed run exercised, so it can be replayed: the seed, and how often
53
+ # each operator came up among the mutations the check picked.
54
+ def report_picked(picked)
55
+ puts " Seed: #{SEED} (MEMORY_CHECK_SEED)"
56
+ return if picked.nil? || picked.empty?
57
+
58
+ tally = picked.map(&:operator_name).tally.sort_by { |operator, count| [-count, operator.to_s] }
59
+ puts " Operators (#{picked.size} picks): #{tally.map { |operator, count| "#{operator} x#{count}" }.join(", ")}"
60
+ end
61
+
41
62
  def format_metrics(result)
42
63
  {
64
+ sustained: result[:sustained_growth_mb] ? format("%.1f MB", result[:sustained_growth_mb]) : "N/A",
43
65
  growth: result[:growth_mb] ? format("%.1f MB", result[:growth_mb]) : "N/A",
44
66
  max: format("%.1f MB", result[:max_growth_kb] / 1024.0),
45
67
  samples: result[:samples].map { |s| s ? format("%.1f", s / 1024.0) : "N/A" }.join(" -> ")
@@ -53,15 +75,21 @@ stub_test_command = ->(_m) { { passed: false } }
53
75
  all_passed = true
54
76
 
55
77
  # 1. InProcess isolation
56
- all_passed &= run_check("InProcess isolation") do
57
- mutation = mutations.sample
78
+ random = Random.new(SEED)
79
+ picked = []
80
+ all_passed &= run_check("InProcess isolation", picked: picked) do
81
+ mutation = mutations.sample(random: random)
82
+ picked << mutation
58
83
  isolator = Evilution::Isolation::InProcess.new
59
84
  isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
60
85
  end
61
86
 
62
87
  # 2. Fork isolation
63
- all_passed &= run_check("Fork isolation") do
64
- mutation = mutations.sample
88
+ random = Random.new(SEED)
89
+ picked = []
90
+ all_passed &= run_check("Fork isolation", picked: picked) do
91
+ mutation = mutations.sample(random: random)
92
+ picked << mutation
65
93
  isolator = Evilution::Isolation::Fork.new
66
94
  isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
67
95
  end
@@ -113,13 +141,31 @@ complex_subjects = complex_parser.call(COMPLEX_FIXTURE)
113
141
  complex_mutations = complex_subjects.flat_map { |s| complex_registry.mutations_for(s) }
114
142
 
115
143
  integration = Evilution::Integration::RSpec.new(test_files: [COMPLEX_FIXTURE_SPEC])
116
-
117
- # Budget is generous: per-mutation require() adds the file to $LOADED_FEATURES and
118
- # accumulates constant-redefinition warnings. Local runs land around 20 MB; CI
119
- # varies up to ~30 MB depending on Ruby build and GC pressure, so we leave headroom.
120
- all_passed &= run_check("RSpec integration per-mutation (Config)", iterations: 20, max_growth_kb: 40_960) do
121
- mutation = complex_mutations.sample
122
- integration.call(mutation)
144
+ complex_pool = complex_mutations.sample(COMPLEX_POOL_SIZE, random: Random.new(SEED))
145
+ complex_turn = complex_pool.cycle
146
+
147
+ # This check used to grow 20-41 MB over 20 mutations, which was put down to
148
+ # per-mutation require() and redefinition warnings and given a 40 MB budget. It
149
+ # was a leak: every run left its examples reachable from
150
+ # RSpec::Core::AnonymousExampleGroup, about 1 MB per mutation (fixed by
151
+ # StateGuard::AnonymousExampleGroupExamples).
152
+ #
153
+ # The first run of a mutation still grows the process, as the heap reaches the
154
+ # size that mutation needs, and the growth depends on which mutation it is. So
155
+ # the check cycles through a fixed pool: warm-up runs each mutation twice, and
156
+ # every measured run repeats work the warm-up already did. Growth from there
157
+ # on is retained per run.
158
+ #
159
+ # The budget is 10 MB over 100 runs, about 100 KB per mutation: a leak of 200 KB
160
+ # per mutation grows about 18 MB once its largest step is taken out, and fails.
161
+ # Measured on Linux (aarch64) over seeds 1, 7, 42 and 99, sustained growth was
162
+ # -0.9 to 0.6 MB on Ruby 3.4.10 and 0.7 to 2.8 MB on 4.0.7. Ruby 4.0.7 creeps
163
+ # by about 30 KB per mutation in RSS with no Ruby objects retained, which is
164
+ # below what this check can tell apart from allocator noise.
165
+ all_passed &= run_check("RSpec integration per-mutation (Config)",
166
+ iterations: COMPLEX_ITERATIONS, max_growth_kb: COMPLEX_MAX_GROWTH_KB,
167
+ warmup_iterations: COMPLEX_POOL_SIZE * 2, picked: complex_pool) do
168
+ integration.call(complex_turn.next)
123
169
  end
124
170
 
125
171
  puts all_passed ? "All memory checks passed." : "Some memory checks failed!"