evilution 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +41 -0
  3. data/CHANGELOG.md +65 -0
  4. data/README.md +149 -13
  5. data/docs/architecture.md +20 -5
  6. data/docs/isolation.md +3 -4
  7. data/exe/evil +8 -1
  8. data/exe/evilution +8 -1
  9. data/lib/evilution/ast/local_reads.rb +43 -0
  10. data/lib/evilution/cli/exit_guard.rb +59 -0
  11. data/lib/evilution/cli/parser/options_builder.rb +1 -0
  12. data/lib/evilution/cli.rb +1 -0
  13. data/lib/evilution/config.rb +3 -2
  14. data/lib/evilution/diagnostic.rb +22 -0
  15. data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
  16. data/lib/evilution/integration/loading/redefinition_recovery.rb +1 -1
  17. data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
  18. data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
  19. data/lib/evilution/integration/minitest.rb +2 -1
  20. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
  21. data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +4 -1
  22. data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
  23. data/lib/evilution/integration/rspec.rb +48 -2
  24. data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
  25. data/lib/evilution/isolation/fork.rb +29 -8
  26. data/lib/evilution/mcp/complete_result_server.rb +41 -0
  27. data/lib/evilution/mcp/server.rb +2 -1
  28. data/lib/evilution/mutator/base.rb +14 -2
  29. data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
  30. data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
  31. data/lib/evilution/mutator/operator/boolean_expression_to_nil.rb +22 -0
  32. data/lib/evilution/mutator/operator/boolean_operand_promotion.rb +31 -0
  33. data/lib/evilution/mutator/operator/case_in.rb +64 -0
  34. data/lib/evilution/mutator/operator/case_when.rb +72 -1
  35. data/lib/evilution/mutator/operator/conditional_branch.rb +20 -7
  36. data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
  37. data/lib/evilution/mutator/operator/if_branch_swap.rb +48 -0
  38. data/lib/evilution/mutator/operator/loop_body_to_raise.rb +68 -0
  39. data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
  40. data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
  41. data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
  42. data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
  43. data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
  44. data/lib/evilution/mutator/operator/pattern_predicate.rb +28 -0
  45. data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
  46. data/lib/evilution/mutator/primitives.rb +52 -0
  47. data/lib/evilution/mutator/registry.rb +14 -0
  48. data/lib/evilution/process_supervisor.rb +20 -8
  49. data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
  50. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
  51. data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
  52. data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
  53. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
  54. data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
  55. data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
  56. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  57. data/lib/evilution/reporter/cli/trailer.rb +11 -7
  58. data/lib/evilution/reporter/cli.rb +20 -4
  59. data/lib/evilution/reporter/json/subjects.rb +29 -0
  60. data/lib/evilution/reporter/json.rb +23 -1
  61. data/lib/evilution/result/mutation_result.rb +3 -2
  62. data/lib/evilution/result/neutral_reason.rb +35 -0
  63. data/lib/evilution/result/subject_score.rb +28 -0
  64. data/lib/evilution/result/subject_scorer.rb +37 -0
  65. data/lib/evilution/result/summary.rb +44 -2
  66. data/lib/evilution/runner/canary.rb +52 -5
  67. data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
  68. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +9 -3
  69. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
  70. data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
  71. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
  72. data/lib/evilution/runner/mutation_executor.rb +32 -4
  73. data/lib/evilution/runner/report_publisher.rb +31 -9
  74. data/lib/evilution/runner/target_spec_audit.rb +43 -0
  75. data/lib/evilution/runner.rb +10 -1
  76. data/lib/evilution/version.rb +1 -1
  77. data/lib/evilution.rb +18 -0
  78. data/scripts/compare_targeting +4 -2
  79. metadata +33 -2
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../line_formatters"
4
+
5
+ # EV-j0bv / GH #1607: parallel workers contending on shared infrastructure — a
6
+ # database file, a lock — crash the test process in a way that says nothing
7
+ # about the mutation. Those mutations are re-run serially once the pool is
8
+ # done, and the run says so, because the alternative is a neutral count that
9
+ # moves with --jobs on identical input with no explanation.
10
+ class Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice
11
+ def format(summary)
12
+ count = summary.infra_retried
13
+ return nil if count.zero?
14
+
15
+ noun = count == 1 ? "mutation" : "mutations"
16
+ pronoun = count == 1 ? "it" : "them"
17
+ "! #{count} #{noun} hit infrastructure errors under parallel workers; re-ran #{pronoun} serially."
18
+ end
19
+ end
@@ -3,18 +3,42 @@
3
3
  require_relative "../line_formatters"
4
4
  require_relative "../pct"
5
5
 
6
+ # The threshold comes from the run's own configuration. Printing a verdict
7
+ # against a default the exit code did not share is what made a 0% score report
8
+ # `FAIL` and exit 0 (EV-39t1 / GH #1604); with no minimum configured the line
9
+ # states the score and says so, rather than implying a gate that is not armed.
6
10
  class Evilution::Reporter::CLI::LineFormatters::ResultLine
7
- DEFAULT_MIN_SCORE = 0.8
8
-
9
- def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: DEFAULT_MIN_SCORE)
11
+ def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: nil)
10
12
  @pct = pct
11
13
  @min_score = min_score
12
14
  end
13
15
 
14
16
  def format(summary)
17
+ return unresolved_targets_line(summary) if summary.unresolved_targets?
18
+ return no_threshold_line(summary) unless gate?
19
+
15
20
  pass_fail = summary.success?(min_score: @min_score) ? "PASS" : "FAIL"
16
21
  score_pct = @pct.format(summary.score)
17
22
  threshold_pct = @pct.format(@min_score)
18
23
  "Result: #{pass_fail} (score #{score_pct} #{pass_fail == "PASS" ? ">=" : "<"} #{threshold_pct})"
19
24
  end
25
+
26
+ private
27
+
28
+ # A minimum of zero passes every score, so it is not a gate either.
29
+ def gate?
30
+ !@min_score.nil? && @min_score.positive?
31
+ end
32
+
33
+ def no_threshold_line(summary)
34
+ "Result: #{@pct.format(summary.score)} (no minimum score set)"
35
+ end
36
+
37
+ # The score covers only the files that resolved to a spec, so reporting it
38
+ # against the threshold here would name the wrong problem.
39
+ def unresolved_targets_line(summary)
40
+ count = summary.unresolved_target_files.length
41
+ subject = count == 1 ? "target file has" : "target files have"
42
+ "Result: FAIL (#{count} #{subject} no resolvable spec)"
43
+ end
20
44
  end
@@ -3,12 +3,30 @@
3
3
  require_relative "../line_formatters"
4
4
  require_relative "../pct"
5
5
 
6
+ # The score covers only the mutations that got a verdict. Where a run left some
7
+ # out — neutral, unresolved, equivalent, errored — full marks over a fraction of
8
+ # it reads as a verdict on the whole, so the line says how much it covered
9
+ # (EV-5pob / GH #1606).
6
10
  class Evilution::Reporter::CLI::LineFormatters::Score
7
11
  def initialize(pct: Evilution::Reporter::CLI::Pct.new)
8
12
  @pct = pct
9
13
  end
10
14
 
11
15
  def format(summary)
12
- "Score: #{@pct.format(summary.score)} (#{summary.killed}/#{summary.score_denominator})"
16
+ "Score: #{@pct.format(summary.score)} (#{counts(summary)})"
17
+ end
18
+
19
+ private
20
+
21
+ def counts(summary)
22
+ verified = summary.score_denominator
23
+ pair = "#{summary.killed}/#{verified}"
24
+ return pair if verified == summary.total
25
+
26
+ "#{pair} verified of #{summary.total} mutations#{neutral_note(summary)}"
27
+ end
28
+
29
+ def neutral_note(summary)
30
+ summary.neutral.positive? ? ", #{summary.neutral} neutral" : ""
13
31
  end
14
32
  end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../line_formatters"
4
+
5
+ # EV-p4sm / GH #1603: a target file that resolves to no test file contributes
6
+ # only a handful of `unresolved` mutations, which UnresolvedRateWarning dilutes
7
+ # against every other file in the run — 14 unresolved out of 98 sits under its
8
+ # threshold and says nothing at all. This formatter reports the stronger,
9
+ # per-file fact instead, and names the files so the reader knows which of the
10
+ # paths they passed was never tested.
11
+ class Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets
12
+ def format(summary)
13
+ return nil unless summary.unresolved_targets?
14
+
15
+ files = summary.unresolved_target_files
16
+ listing = files.map { |path| " #{path}" }.join("\n")
17
+ "! #{headline(files.length, summary.target_file_count)}:\n#{listing}"
18
+ end
19
+
20
+ private
21
+
22
+ # The ratio is dropped rather than guessed when the run's target count is
23
+ # unknown, which is how a summary restored from a saved session arrives. The
24
+ # noun agrees with whichever number it follows ("1 of 2 target files has",
25
+ # "1 target file has"), the verb with how many were unresolved.
26
+ def headline(unresolved_count, target_file_count)
27
+ noun_count = target_file_count.nil? ? unresolved_count : target_file_count
28
+ noun = noun_count == 1 ? "target file" : "target files"
29
+ verb = unresolved_count == 1 ? "has" : "have"
30
+ tail = unresolved_count == 1 ? "it was never tested" : "they were never tested"
31
+ scope = target_file_count.nil? ? unresolved_count.to_s : "#{unresolved_count} of #{target_file_count}"
32
+
33
+ "#{scope} #{noun} #{verb} no resolvable spec — #{tail}"
34
+ end
35
+ end
@@ -5,6 +5,8 @@ require_relative "line_formatters/mutations"
5
5
  require_relative "line_formatters/score"
6
6
  require_relative "line_formatters/error_rate_warning"
7
7
  require_relative "line_formatters/unresolved_rate_warning"
8
+ require_relative "line_formatters/unresolved_targets"
9
+ require_relative "line_formatters/infra_retry_notice"
8
10
  require_relative "line_formatters/duration"
9
11
  require_relative "line_formatters/efficiency"
10
12
  require_relative "line_formatters/peak_memory"
@@ -15,6 +17,8 @@ class Evilution::Reporter::CLI::MetricsBlock
15
17
  Evilution::Reporter::CLI::LineFormatters::Score.new,
16
18
  Evilution::Reporter::CLI::LineFormatters::ErrorRateWarning.new,
17
19
  Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning.new,
20
+ Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets.new,
21
+ Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice.new,
18
22
  Evilution::Reporter::CLI::LineFormatters::Duration.new,
19
23
  Evilution::Reporter::CLI::LineFormatters::Efficiency.new,
20
24
  Evilution::Reporter::CLI::LineFormatters::PeakMemory.new
@@ -6,14 +6,18 @@ require_relative "line_formatters/result_line"
6
6
  require_relative "line_formatters/feedback_footer"
7
7
 
8
8
  class Evilution::Reporter::CLI::Trailer
9
- DEFAULT_LINES = [
10
- Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
11
- Evilution::Reporter::CLI::LineFormatters::ResultLine.new,
12
- Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
13
- ].freeze
9
+ def self.default_lines(min_score: nil)
10
+ [
11
+ Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
12
+ Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score),
13
+ Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
14
+ ]
15
+ end
16
+
17
+ DEFAULT_LINES = default_lines.freeze
14
18
 
15
- def initialize(lines: DEFAULT_LINES)
16
- @lines = lines
19
+ def initialize(lines: nil, min_score: nil)
20
+ @lines = lines || self.class.default_lines(min_score: min_score)
17
21
  end
18
22
 
19
23
  def call(summary)
@@ -6,11 +6,12 @@ class Evilution::Reporter::CLI
6
6
  SEPARATOR = "=" * 44
7
7
 
8
8
  def initialize(
9
+ min_score: nil,
9
10
  header: LineFormatters::Header.new,
10
11
  metrics_block: MetricsBlock.new,
11
12
  section_renderer: SectionRenderer.new,
12
13
  sections: DEFAULT_SECTIONS,
13
- trailer: Trailer.new
14
+ trailer: Trailer.new(min_score: min_score)
14
15
  )
15
16
  @header = header
16
17
  @metrics_block = metrics_block
@@ -46,6 +47,9 @@ require_relative "cli/line_formatters/result_line"
46
47
  require_relative "cli/line_formatters/feedback_footer"
47
48
  require_relative "cli/item_formatters/coverage_gap"
48
49
  require_relative "cli/item_formatters/result_location"
50
+ require_relative "cli/item_formatters/neutral_group"
51
+ require_relative "cli/item_formatters/subject_score"
52
+ require_relative "cli/item_formatters/subject_score_group"
49
53
  require_relative "cli/item_formatters/error"
50
54
  require_relative "cli/item_formatters/disabled"
51
55
  require_relative "cli/metrics_block"
@@ -60,9 +64,21 @@ Evilution::Reporter::CLI.const_set(
60
64
  formatter: Evilution::Reporter::CLI::ItemFormatters::CoverageGap.new
61
65
  ),
62
66
  Evilution::Reporter::CLI::Section.new(
63
- title: "Neutral mutations (test already failing):",
64
- fetcher: lambda(&:neutral_results),
65
- formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
67
+ title: lambda { |groups|
68
+ subjects = groups.sum(&:length)
69
+ "Subjects needing attention (#{subjects} subject#{"s" unless subjects == 1} " \
70
+ "in #{groups.length} file#{"s" unless groups.length == 1}):"
71
+ },
72
+ fetcher: lambda(&:subjects_needing_attention_by_file),
73
+ formatter: Evilution::Reporter::CLI::ItemFormatters::SubjectScoreGroup.new
74
+ ),
75
+ Evilution::Reporter::CLI::Section.new(
76
+ title: lambda { |groups|
77
+ count = groups.sum { |(_reason, results)| results.length }
78
+ "Neutral mutations (#{count}, not verified):"
79
+ },
80
+ fetcher: lambda(&:neutral_results_by_reason),
81
+ formatter: Evilution::Reporter::CLI::ItemFormatters::NeutralGroup.new
66
82
  ),
67
83
  Evilution::Reporter::CLI::Section.new(
68
84
  title: "Equivalent mutations (provably identical behavior):",
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../json"
4
+
5
+ # The per-subject rows of a JSON report.
6
+ #
7
+ # Every subject is listed, whether or not it needs attention: a consumer that
8
+ # wants only the gaps filters on `reached` and `score`, while one asserting that
9
+ # a method is covered at all needs to see the rest (EV-nlx1 / GH #1605).
10
+ class Evilution::Reporter::JSON::Subjects
11
+ def call(summary)
12
+ summary.subject_scores.map { |score| row(score) }
13
+ end
14
+
15
+ private
16
+
17
+ def row(score)
18
+ {
19
+ name: score.name,
20
+ file: score.file_path,
21
+ total: score.total,
22
+ killed: score.killed,
23
+ verified: score.verified,
24
+ survived: score.survived,
25
+ score: score.score.round(4),
26
+ reached: score.reached?
27
+ }
28
+ end
29
+ end
@@ -8,8 +8,9 @@ require_relative "../reporter"
8
8
  require_relative "../session/schema"
9
9
 
10
10
  class Evilution::Reporter::JSON
11
- def initialize(suggest_tests: false, integration: :rspec)
11
+ def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new)
12
12
  @suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
13
+ @subjects = subjects
13
14
  end
14
15
 
15
16
  def call(summary)
@@ -25,6 +26,7 @@ class Evilution::Reporter::JSON
25
26
  timestamp: Time.now.iso8601,
26
27
  summary: build_summary(summary),
27
28
  coverage_gaps: build_coverage_gaps(summary),
29
+ subjects: @subjects.call(summary),
28
30
  **result_categories(summary)
29
31
  }
30
32
  append_disabled_to_report(report, summary)
@@ -94,12 +96,20 @@ class Evilution::Reporter::JSON
94
96
  end
95
97
 
96
98
  def append_optional_summary_fields(data, summary)
99
+ append_diagnostic_summary_fields(data, summary)
97
100
  data[:truncated] = true if summary.truncated?
98
101
  data[:skipped] = summary.skipped if summary.skipped.positive?
99
102
  peak = summary.peak_memory_mb
100
103
  data[:peak_memory_mb] = peak.round(1) if peak
101
104
  end
102
105
 
106
+ # What the run had to say about itself rather than about the mutations: a
107
+ # target that was never tested, work that had to be redone serially.
108
+ def append_diagnostic_summary_fields(data, summary)
109
+ data[:unresolved_target_files] = summary.unresolved_target_files if summary.unresolved_targets?
110
+ data[:infra_retried] = summary.infra_retried if summary.infra_retried.positive?
111
+ end
112
+
103
113
  def build_mutation_detail(result)
104
114
  mutation = result.mutation
105
115
  detail = base_mutation_fields(mutation, result)
@@ -107,9 +117,19 @@ class Evilution::Reporter::JSON
107
117
  detail[:test_command] = result.test_command if result.test_command
108
118
  append_memory_fields(detail, result)
109
119
  append_error_fields(detail, result)
120
+ append_neutral_reason(detail, result)
110
121
  detail
111
122
  end
112
123
 
124
+ # Why a neutral was recorded — a red baseline and an infrastructure crash want
125
+ # opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
126
+ def append_neutral_reason(detail, result)
127
+ reason = result.neutral_reason
128
+ return unless reason
129
+
130
+ detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
131
+ end
132
+
113
133
  def base_mutation_fields(mutation, result)
114
134
  {
115
135
  operator: mutation.operator_name,
@@ -161,3 +181,5 @@ class Evilution::Reporter::JSON
161
181
  }
162
182
  end
163
183
  end
184
+
185
+ require_relative "json/subjects"
@@ -7,10 +7,10 @@ require_relative "memory_stats"
7
7
  class Evilution::Result::MutationResult
8
8
  STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
9
9
 
10
- attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error
10
+ attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
11
11
 
12
12
  def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
13
- test_command: nil, memory: nil, error: nil)
13
+ test_command: nil, memory: nil, error: nil, neutral_reason: nil)
14
14
  raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
15
15
 
16
16
  @mutation = mutation
@@ -20,6 +20,7 @@ class Evilution::Result::MutationResult
20
20
  @test_command = test_command
21
21
  @memory = memory
22
22
  @error = error
23
+ @neutral_reason = neutral_reason
23
24
  freeze
24
25
  end
25
26
 
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # Why a mutation was recorded neutral.
6
+ #
7
+ # Neutral covers two unrelated situations, and they call for opposite responses:
8
+ # a spec file that was already red before any mutation ran, and a test process
9
+ # that died on infrastructure rather than on the mutation. Without the reason
10
+ # they are indistinguishable in a report, and a run can print full marks while
11
+ # the neutral bucket quietly holds what would otherwise be survivors
12
+ # (EV-5pob / GH #1606).
13
+ Evilution::Result::NeutralReason = Data.define(:kind, :detail) do
14
+ def self.baseline_failure(spec_file)
15
+ new(kind: :baseline_failure, detail: spec_file)
16
+ end
17
+
18
+ def self.infra_error(error_class)
19
+ new(kind: :infra_error, detail: error_class)
20
+ end
21
+
22
+ def to_s
23
+ detail ? "#{label} (#{detail})" : label
24
+ end
25
+
26
+ private
27
+
28
+ def label
29
+ case kind
30
+ when :baseline_failure then "baseline already failing"
31
+ when :infra_error then "infrastructure error"
32
+ else kind.to_s
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # What one subject — a single method — scored, alongside the counts behind it.
6
+ #
7
+ # The run's own score is computed per file, so a well-tested file that gains new
8
+ # untested methods still reports 100%: whatever the resolved spec reaches is all
9
+ # the number ever describes (EV-nlx1 / GH #1605). Per subject the picture
10
+ # separates: a method whose mutations were never reached scores nothing and says
11
+ # so, instead of disappearing into the file's total.
12
+ Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived) do
13
+ # Killed over what actually got a verdict, the same denominator the run uses.
14
+ def score
15
+ return 0.0 if verified.zero?
16
+
17
+ killed.to_f / verified
18
+ end
19
+
20
+ # Whether any mutation of this subject got a verdict at all.
21
+ def reached?
22
+ verified.positive?
23
+ end
24
+
25
+ def fully_verified?
26
+ reached? && killed == verified
27
+ end
28
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "subject_score"
4
+
5
+ # Groups mutation results by the subject they belong to and scores each one.
6
+ #
7
+ # Same denominator as the run's own score: statuses that carry no verdict
8
+ # (unresolved, neutral, equivalent, errored, unparseable) are left out of it, so
9
+ # a subject those account for entirely reports as unreached rather than as a
10
+ # perfect score over nothing.
11
+ class Evilution::Result::SubjectScorer
12
+ VERDICT_STATUSES = %i[killed survived timeout].freeze
13
+ private_constant :VERDICT_STATUSES
14
+
15
+ def call(results)
16
+ grouped = results.group_by { |result| [result.mutation.file_path, result.mutation.subject.name] }
17
+
18
+ grouped
19
+ .map { |(file_path, name), subject_results| score_for(name, file_path, subject_results) }
20
+ .sort_by { |score| [score.file_path, score.name] }
21
+ end
22
+
23
+ private
24
+
25
+ def score_for(name, file_path, results)
26
+ verdicts = results.count { |result| VERDICT_STATUSES.include?(result.status) }
27
+
28
+ Evilution::Result::SubjectScore.new(
29
+ name: name,
30
+ file_path: file_path,
31
+ total: results.length,
32
+ killed: results.count(&:killed?),
33
+ verified: verdicts,
34
+ survived: results.count(&:survived?)
35
+ )
36
+ end
37
+ end
@@ -2,19 +2,57 @@
2
2
 
3
3
  require_relative "../result"
4
4
  require_relative "coverage_gap_grouper"
5
+ require_relative "subject_scorer"
5
6
 
6
7
  class Evilution::Result::Summary
7
- attr_reader :results, :duration, :skipped, :disabled_mutations
8
+ attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
9
+ :target_file_count, :infra_retried
8
10
 
9
- def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [])
11
+ def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
12
+ unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
10
13
  @results = results
11
14
  @duration = duration
12
15
  @truncated = truncated
13
16
  @skipped = skipped
14
17
  @disabled_mutations = disabled_mutations
18
+ @unresolved_target_files = unresolved_target_files.freeze
19
+ @target_file_count = target_file_count
20
+ @infra_retried = infra_retried
15
21
  freeze
16
22
  end
17
23
 
24
+ # Neutral results gathered by the reason they were recorded, which is what the
25
+ # report needs to say which kind of neutral a reader is looking at
26
+ # (EV-5pob / GH #1606).
27
+ def neutral_results_by_reason
28
+ neutral_results.group_by(&:neutral_reason).to_a
29
+ end
30
+
31
+ # What each subject — each method — scored on its own. The run's score is
32
+ # computed per file, which says nothing about a method inside it that no
33
+ # example reaches (EV-nlx1 / GH #1605).
34
+ def subject_scores
35
+ Evilution::Result::SubjectScorer.new.call(results)
36
+ end
37
+
38
+ # The subjects the file-level score does not speak for: something survived, or
39
+ # nothing reached them at all.
40
+ def subjects_needing_attention
41
+ subject_scores.reject(&:fully_verified?)
42
+ end
43
+
44
+ # The same subjects, gathered under the file they live in, which is how the
45
+ # report lists them.
46
+ def subjects_needing_attention_by_file
47
+ subjects_needing_attention.group_by(&:file_path).values
48
+ end
49
+
50
+ # Files evilution was pointed at that resolved to no test file, so nothing
51
+ # about them was ever measured (EV-p4sm / GH #1603).
52
+ def unresolved_targets?
53
+ !unresolved_target_files.empty?
54
+ end
55
+
18
56
  def truncated?
19
57
  @truncated
20
58
  end
@@ -66,7 +104,11 @@ class Evilution::Result::Summary
66
104
  killed.to_f / denominator
67
105
  end
68
106
 
107
+ # A target file that was never tested fails the run on its own: the score
108
+ # only speaks for the files that did resolve to a spec.
69
109
  def success?(min_score: 1.0)
110
+ return false if unresolved_targets?
111
+
70
112
  score >= min_score
71
113
  end
72
114
 
@@ -34,7 +34,7 @@ class Evilution::Runner::Canary
34
34
  test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
35
35
  timeout: @config.timeout
36
36
  )
37
- raise Failed, failure_message(result.status) unless result.status == :survived
37
+ raise Failed, failure_message(result) unless result.status == :survived
38
38
 
39
39
  nil
40
40
  ensure
@@ -146,12 +146,59 @@ class Evilution::Runner::Canary
146
146
  @integration_class.new(test_files: [spec_path], hooks: @hooks)
147
147
  end
148
148
 
149
- def failure_message(status)
149
+ # When the child reported an error, that error IS the diagnosis -- naming it
150
+ # beats guessing. The speculative list stays only for the cases that carry no
151
+ # error at all (:killed, :timeout), where guesses are the only help there is.
152
+ # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
153
+ # this message used to drop, and none of the four guesses was the cause.
154
+ # EV-65nf / GH #1586.
155
+ def failure_message(result)
156
+ "#{failure_preamble(result.status)} #{diagnosis(result)} " \
157
+ "Re-run with --no-canary to bypass this check."
158
+ end
159
+
160
+ def failure_preamble(status)
150
161
  "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
151
162
  "mutation was scored #{status.inspect} instead of :survived. The mutation " \
152
- "pipeline is misreporting — every score this run would produce is unreliable. " \
153
- "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
163
+ "pipeline is misreporting — every score this run would produce is unreliable."
164
+ end
165
+
166
+ def diagnosis(result)
167
+ reported = reported_error(result)
168
+ return speculative_causes if reported.nil?
169
+
170
+ "The child reported: #{reported}"
171
+ end
172
+
173
+ def reported_error(result)
174
+ message = result.error_message
175
+ return nil if message.nil? || message.empty?
176
+
177
+ described = with_class_prefix(message, result.error_class)
178
+ frame = first_frame(result)
179
+ frame.nil? ? described : "#{described} (at #{frame})"
180
+ end
181
+
182
+ # MutationApplier already prefixes the class onto the message it packs
183
+ # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
184
+ # leave the class in its own field. Prefixing unconditionally would print
185
+ # "NameError: NameError: ..." for the former.
186
+ def with_class_prefix(message, klass)
187
+ return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
188
+
189
+ "#{klass}: #{message}"
190
+ end
191
+
192
+ def first_frame(result)
193
+ backtrace = result.error_backtrace
194
+ return nil if backtrace.nil? || backtrace.empty?
195
+
196
+ backtrace.first
197
+ end
198
+
199
+ def speculative_causes
200
+ "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
154
201
  "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
155
- "an isolation-mode defect. Re-run with --no-canary to bypass this check."
202
+ "an isolation-mode defect."
156
203
  end
157
204
  end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../mutation_executor"
4
+ require_relative "neutralizer/infra_error"
5
+
6
+ # Re-runs, one at a time, the mutations a parallel pass could not judge because
7
+ # the test process crashed on infrastructure — a database lock, a statement
8
+ # timeout — rather than on the mutation itself.
9
+ #
10
+ # Those crashes are demoted to `:neutral` so they do not inflate the kill count
11
+ # (EV-toid / GH #814), which leaves the neutral bucket swinging with `--jobs`
12
+ # on identical input: the contention that causes them only exists while several
13
+ # workers run at once. Re-running them after the pool is done removes the
14
+ # contention, so the verdict matches what a serial run would have produced
15
+ # (EV-j0bv / GH #1607).
16
+ #
17
+ # Only mutations neutralised by an infra crash are re-run. A neutral from a
18
+ # failing baseline is a real statement about the spec, and re-running it would
19
+ # say the same thing again.
20
+ class Evilution::Runner::MutationExecutor::InfraRetry
21
+ attr_reader :retried_count
22
+
23
+ def initialize(runner:, pipeline:)
24
+ @runner = runner
25
+ @pipeline = pipeline
26
+ @retried_count = 0
27
+ end
28
+
29
+ def call(results, baseline_result:, integration:)
30
+ @retried_count = 0
31
+
32
+ results.map do |result|
33
+ next result unless retryable?(result)
34
+
35
+ @retried_count += 1
36
+ rerun(result, baseline_result: baseline_result, integration: integration)
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def retryable?(result)
43
+ Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_neutral?(result)
44
+ end
45
+
46
+ # The parallel pass keeps the sources of a retryable mutation alive so it can
47
+ # be applied again; they are released here, once it has been.
48
+ def rerun(result, baseline_result:, integration:)
49
+ mutation = result.mutation
50
+ rerun_result = @runner.call(mutation, integration: integration)
51
+ mutation.strip_sources!
52
+ @pipeline.call(rerun_result, baseline_result: baseline_result)
53
+ end
54
+ end