evilution 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +1 -0
  3. data/.rubocop_todo.yml +30 -0
  4. data/CHANGELOG.md +59 -0
  5. data/README.md +285 -45
  6. data/docs/architecture.md +110 -12
  7. data/docs/ast_pattern_syntax.md +35 -7
  8. data/docs/combined_runs_spike.md +407 -0
  9. data/docs/integrations.md +1 -1
  10. data/docs/isolation.md +63 -2
  11. data/docs/migration-from-mutant.md +16 -11
  12. data/docs/mutant_schemata_spike.md +387 -0
  13. data/docs/operator_yield_study.md +576 -0
  14. data/docs/probe_first_spike.md +264 -0
  15. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  16. data/lib/evilution/ast/callback_declaration.rb +36 -3
  17. data/lib/evilution/ast/class_body.rb +62 -16
  18. data/lib/evilution/ast/def_region.rb +130 -0
  19. data/lib/evilution/ast/local_declaration.rb +37 -0
  20. data/lib/evilution/ast/parent_index.rb +50 -0
  21. data/lib/evilution/ast/parse_cache.rb +27 -0
  22. data/lib/evilution/ast/parser.rb +8 -3
  23. data/lib/evilution/ast/pattern/filter.rb +53 -12
  24. data/lib/evilution/ast/skip_filter.rb +83 -0
  25. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  26. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  27. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  28. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  29. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  30. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  31. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  32. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  33. data/lib/evilution/ast/skip_sites.rb +9 -0
  34. data/lib/evilution/ast/source_surgeon.rb +20 -2
  35. data/lib/evilution/ast/syntax_check.rb +10 -8
  36. data/lib/evilution/baseline/report.rb +11 -2
  37. data/lib/evilution/baseline.rb +31 -6
  38. data/lib/evilution/cache/keys.rb +140 -0
  39. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  40. data/lib/evilution/cache/skeleton.rb +57 -0
  41. data/lib/evilution/cache/spec_digest.rb +39 -0
  42. data/lib/evilution/cache.rb +145 -42
  43. data/lib/evilution/class_tree.rb +38 -0
  44. data/lib/evilution/cli/command.rb +12 -4
  45. data/lib/evilution/cli/commands/run.rb +12 -3
  46. data/lib/evilution/cli/commands/subjects.rb +5 -2
  47. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  48. data/lib/evilution/cli/printers/environment.rb +34 -4
  49. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  50. data/lib/evilution/compare/categorizer.rb +2 -1
  51. data/lib/evilution/compare/normalizer.rb +3 -2
  52. data/lib/evilution/config/sources.rb +3 -0
  53. data/lib/evilution/config/validators/base.rb +9 -0
  54. data/lib/evilution/config/validators/since.rb +22 -0
  55. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  56. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  58. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  59. data/lib/evilution/config.rb +122 -6
  60. data/lib/evilution/coverage/load_tracker.rb +50 -0
  61. data/lib/evilution/coverage/map.rb +42 -5
  62. data/lib/evilution/coverage/map_builder.rb +123 -22
  63. data/lib/evilution/coverage/map_store.rb +15 -5
  64. data/lib/evilution/coverage/recorder.rb +42 -3
  65. data/lib/evilution/coverage_example_filter.rb +35 -0
  66. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  67. data/lib/evilution/example_filter.rb +20 -0
  68. data/lib/evilution/git/changed_lines.rb +103 -0
  69. data/lib/evilution/git/since_scope.rb +77 -0
  70. data/lib/evilution/git/unified_diff.rb +68 -0
  71. data/lib/evilution/infra_crash.rb +26 -0
  72. data/lib/evilution/integration/base.rb +97 -6
  73. data/lib/evilution/integration/crash_detector.rb +46 -6
  74. data/lib/evilution/integration/known_failures.rb +4 -0
  75. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  76. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  77. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  78. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  79. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  80. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  81. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  82. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  83. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  84. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  85. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  86. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  87. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  88. data/lib/evilution/integration/minitest.rb +22 -2
  89. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  90. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  91. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  92. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  93. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  94. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  95. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  96. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  97. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  98. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  99. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  100. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  101. data/lib/evilution/integration/rspec.rb +98 -47
  102. data/lib/evilution/integration/stop_policy.rb +49 -0
  103. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  104. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  105. data/lib/evilution/integration/test_unit.rb +28 -2
  106. data/lib/evilution/isolation/fork.rb +58 -13
  107. data/lib/evilution/isolation/in_process.rb +12 -1
  108. data/lib/evilution/line_set.rb +72 -0
  109. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  110. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  111. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  112. data/lib/evilution/mcp/info_tool.rb +24 -17
  113. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  114. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  115. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  116. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  117. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  118. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  119. data/lib/evilution/mcp/session_tool.rb +1 -2
  120. data/lib/evilution/mcp.rb +4 -0
  121. data/lib/evilution/mutation/line_diff.rb +111 -0
  122. data/lib/evilution/mutation.rb +17 -13
  123. data/lib/evilution/mutator/base.rb +121 -98
  124. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  125. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  126. data/lib/evilution/mutator/load_sources.rb +88 -0
  127. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  128. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  129. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  130. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  131. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  132. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  133. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  135. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  136. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  137. data/lib/evilution/mutator/registry.rb +11 -5
  138. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  139. data/lib/evilution/parallel/pool.rb +11 -4
  140. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  141. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  142. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  143. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  144. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  145. data/lib/evilution/parallel/work_queue.rb +69 -12
  146. data/lib/evilution/parallel_db_warning.rb +39 -12
  147. data/lib/evilution/process_supervisor.rb +22 -2
  148. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  149. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  150. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  151. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  152. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  153. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  154. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  155. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  156. data/lib/evilution/reporter/cli.rb +9 -2
  157. data/lib/evilution/reporter/html/assets/style.css +8 -0
  158. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  159. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  160. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  161. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  162. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  163. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  164. data/lib/evilution/reporter/json/subjects.rb +1 -0
  165. data/lib/evilution/reporter/json/timings.rb +42 -0
  166. data/lib/evilution/reporter/json.rb +13 -50
  167. data/lib/evilution/result/coverage_gap.rb +5 -0
  168. data/lib/evilution/result/mutation_result.rb +55 -3
  169. data/lib/evilution/result/mutation_timings.rb +54 -0
  170. data/lib/evilution/result/run_timings.rb +20 -0
  171. data/lib/evilution/result/subject_score.rb +8 -1
  172. data/lib/evilution/result/subject_scorer.rb +5 -3
  173. data/lib/evilution/result/summary.rb +49 -12
  174. data/lib/evilution/runner/baseline_runner.rb +47 -5
  175. data/lib/evilution/runner/diagnostics.rb +0 -13
  176. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  177. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  178. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  179. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  180. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  181. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  182. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  183. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  184. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  185. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  186. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  187. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  190. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  191. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  195. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  196. data/lib/evilution/runner/mutation_executor.rb +75 -10
  197. data/lib/evilution/runner/mutation_planner.rb +29 -21
  198. data/lib/evilution/runner/phase_clock.rb +22 -0
  199. data/lib/evilution/runner/report_publisher.rb +1 -1
  200. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  201. data/lib/evilution/runner.rb +87 -26
  202. data/lib/evilution/session/diff.rb +9 -2
  203. data/lib/evilution/session/store.rb +17 -2
  204. data/lib/evilution/source_ast_cache.rb +10 -0
  205. data/lib/evilution/spec_ast_cache.rb +15 -0
  206. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  207. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  208. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  209. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  210. data/lib/evilution/spec_resolver.rb +76 -65
  211. data/lib/evilution/spec_selector.rb +17 -13
  212. data/lib/evilution/subject.rb +7 -2
  213. data/lib/evilution/version.rb +1 -1
  214. data/lib/evilution/worker_database/copies.rb +73 -0
  215. data/lib/evilution/worker_database/schema.rb +39 -0
  216. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  217. data/lib/evilution/worker_database.rb +214 -0
  218. data/schema/evilution.config.schema.json +74 -2
  219. metadata +84 -13
  220. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  221. data/scripts/benchmark_density +0 -262
  222. data/scripts/benchmark_density.yml +0 -19
  223. data/scripts/canary_manifest.yml +0 -47
  224. data/scripts/compare_mutations +0 -421
  225. data/scripts/compare_mutations.yml +0 -24
  226. data/scripts/compare_targeting +0 -279
  227. data/scripts/compare_targeting.example.yml +0 -24
  228. data/scripts/mutant_json_adapter +0 -227
@@ -8,13 +8,17 @@ require_relative "../pct"
8
8
  # `FAIL` and exit 0 (EV-39t1 / GH #1604); with no minimum configured the line
9
9
  # states the score and says so, rather than implying a gate that is not armed.
10
10
  class Evilution::Reporter::CLI::LineFormatters::ResultLine
11
- def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: nil)
11
+ # since: the git ref a --since run was limited to the changes since. Such a
12
+ # run that mutated nothing passes: its changes hold nothing to score.
13
+ def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: nil, since: nil)
12
14
  @pct = pct
13
15
  @min_score = min_score
16
+ @since = since
14
17
  end
15
18
 
16
19
  def format(summary)
17
20
  return unresolved_targets_line(summary) if summary.unresolved_targets?
21
+ return "Result: PASS (nothing to mutate in the lines changed since #{@since})" if nothing_changed?(summary)
18
22
  return no_threshold_line(summary) unless gate?
19
23
 
20
24
  pass_fail = summary.success?(min_score: @min_score) ? "PASS" : "FAIL"
@@ -25,6 +29,10 @@ class Evilution::Reporter::CLI::LineFormatters::ResultLine
25
29
 
26
30
  private
27
31
 
32
+ def nothing_changed?(summary)
33
+ !@since.nil? && summary.total.zero?
34
+ end
35
+
28
36
  # A minimum of zero passes every score, so it is not a gate either.
29
37
  def gate?
30
38
  !@min_score.nil? && @min_score.positive?
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../line_formatters"
4
+
5
+ # The mutations each skip_sites preset left out, under the mutation count
6
+ # whose "skipped" figure they are part of.
7
+ class Evilution::Reporter::CLI::LineFormatters::SkippedSites
8
+ def format(summary)
9
+ sites = summary.skipped_sites
10
+ return nil if sites.empty?
11
+
12
+ "Skipped sites: #{sites.map { |name, count| "#{name} #{count}" }.join(", ")}"
13
+ end
14
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../line_formatters"
4
+
5
+ # A run aimed at lines that hold no subject generates nothing for them, and
6
+ # its report alone reads like a run that found nothing to mutate. The warning
7
+ # printed while the run starts says why, but it is easily lost above the
8
+ # progress output; this line repeats where in the report itself, next to the
9
+ # mutation count it explains.
10
+ class Evilution::Reporter::CLI::LineFormatters::UncoveredCode
11
+ def format(summary)
12
+ return nil unless summary.uncovered_code?
13
+
14
+ listing = summary.uncovered_code.map { |entry| " #{entry[:file]}:#{entry[:lines].join(", ")}" }.join("\n")
15
+ "! Targeted lines hold code outside every subject — no mutations were generated for them:\n#{listing}"
16
+ end
17
+ end
@@ -2,11 +2,13 @@
2
2
 
3
3
  require_relative "../cli"
4
4
  require_relative "line_formatters/mutations"
5
+ require_relative "line_formatters/skipped_sites"
5
6
  require_relative "line_formatters/score"
6
7
  require_relative "line_formatters/baseline_neutralized_notice"
7
8
  require_relative "line_formatters/error_rate_warning"
8
9
  require_relative "line_formatters/unresolved_rate_warning"
9
10
  require_relative "line_formatters/unresolved_targets"
11
+ require_relative "line_formatters/uncovered_code"
10
12
  require_relative "line_formatters/infra_retry_notice"
11
13
  require_relative "line_formatters/duration"
12
14
  require_relative "line_formatters/efficiency"
@@ -15,11 +17,13 @@ require_relative "line_formatters/peak_memory"
15
17
  class Evilution::Reporter::CLI::MetricsBlock
16
18
  DEFAULT_LINES = [
17
19
  Evilution::Reporter::CLI::LineFormatters::Mutations.new,
20
+ Evilution::Reporter::CLI::LineFormatters::SkippedSites.new,
18
21
  Evilution::Reporter::CLI::LineFormatters::Score.new,
19
22
  Evilution::Reporter::CLI::LineFormatters::BaselineNeutralizedNotice.new,
20
23
  Evilution::Reporter::CLI::LineFormatters::ErrorRateWarning.new,
21
24
  Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning.new,
22
25
  Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets.new,
26
+ Evilution::Reporter::CLI::LineFormatters::UncoveredCode.new,
23
27
  Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice.new,
24
28
  Evilution::Reporter::CLI::LineFormatters::Duration.new,
25
29
  Evilution::Reporter::CLI::LineFormatters::Efficiency.new,
@@ -6,18 +6,18 @@ require_relative "line_formatters/result_line"
6
6
  require_relative "line_formatters/feedback_footer"
7
7
 
8
8
  class Evilution::Reporter::CLI::Trailer
9
- def self.default_lines(min_score: nil)
9
+ def self.default_lines(min_score: nil, since: nil)
10
10
  [
11
11
  Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
12
- Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score),
12
+ Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score, since: since),
13
13
  Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
14
14
  ]
15
15
  end
16
16
 
17
17
  DEFAULT_LINES = default_lines.freeze
18
18
 
19
- def initialize(lines: nil, min_score: nil)
20
- @lines = lines || self.class.default_lines(min_score: min_score)
19
+ def initialize(lines: nil, min_score: nil, since: nil)
20
+ @lines = lines || self.class.default_lines(min_score: min_score, since: since)
21
21
  end
22
22
 
23
23
  def call(summary)
@@ -7,11 +7,12 @@ class Evilution::Reporter::CLI
7
7
 
8
8
  def initialize(
9
9
  min_score: nil,
10
+ since: nil,
10
11
  header: LineFormatters::Header.new,
11
12
  metrics_block: MetricsBlock.new,
12
13
  section_renderer: SectionRenderer.new,
13
14
  sections: DEFAULT_SECTIONS,
14
- trailer: Trailer.new(min_score: min_score)
15
+ trailer: Trailer.new(min_score: min_score, since: since)
15
16
  )
16
17
  @header = header
17
18
  @metrics_block = metrics_block
@@ -38,6 +39,7 @@ require_relative "cli/section"
38
39
  require_relative "cli/section_renderer"
39
40
  require_relative "cli/line_formatters/header"
40
41
  require_relative "cli/line_formatters/mutations"
42
+ require_relative "cli/line_formatters/skipped_sites"
41
43
  require_relative "cli/line_formatters/score"
42
44
  require_relative "cli/line_formatters/duration"
43
45
  require_relative "cli/line_formatters/efficiency"
@@ -60,9 +62,14 @@ Evilution::Reporter::CLI.const_set(
60
62
  [
61
63
  Evilution::Reporter::CLI::Section.new(
62
64
  title: ->(gaps) { "Survived mutations (#{gaps.length} coverage gap#{"s" unless gaps.length == 1}):" },
63
- fetcher: lambda(&:coverage_gaps),
65
+ fetcher: lambda(&:survived_gaps),
64
66
  formatter: Evilution::Reporter::CLI::ItemFormatters::CoverageGap.new
65
67
  ),
68
+ Evilution::Reporter::CLI::Section.new(
69
+ title: "Uncovered mutations (no test executes the lines they change, not run):",
70
+ fetcher: lambda(&:uncovered_results),
71
+ formatter: Evilution::Reporter::CLI::ItemFormatters::ResultLocation.new
72
+ ),
66
73
  Evilution::Reporter::CLI::Section.new(
67
74
  title: lambda { |groups|
68
75
  subjects = groups.sum(&:length)
@@ -28,6 +28,7 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
28
28
  .map-line.equivalent { color: #8b949e; }
29
29
  .map-line.unresolved { color: #8b949e; }
30
30
  .map-line.unparseable { color: #8b949e; }
31
+ .map-line.uncovered { color: #f85149; }
31
32
  .line-number { min-width: 60px; color: #8b949e; }
32
33
  .operator { flex: 1; }
33
34
  .status-badge { font-size: 0.7rem; padding: 0.1rem 0.5rem; border-radius: 10px; text-transform: uppercase; font-weight: bold; }
@@ -39,6 +40,7 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
39
40
  .status-badge.error { background: #4a3a10; }
40
41
  .status-badge.unresolved { background: #21262d; }
41
42
  .status-badge.unparseable { background: #21262d; }
43
+ .status-badge.uncovered { background: #4a1a1a; }
42
44
  .survived-details { border-top: 1px solid #30363d; padding: 1rem; }
43
45
  .survived-details h3 { color: #f85149; font-size: 0.9rem; margin-bottom: 0.75rem; }
44
46
  .survived-entry { background: #1c1a1a; border: 1px solid #4a1a1a; border-radius: 6px; padding: 0.75rem; margin-bottom: 0.75rem; }
@@ -83,3 +85,9 @@ h1 { font-size: 1.5rem; color: #f0f6fc; }
83
85
  .survived-entry.regression { border-color: #f85149; background: #2a1010; }
84
86
  .regression-badge { background: #da3633; color: #fff; font-size: 0.65rem; padding: 0.1rem 0.4rem; border-radius: 4px; margin-left: 0.5rem; text-transform: uppercase; font-weight: bold; }
85
87
  footer { text-align: center; color: #484f58; font-size: 0.75rem; margin-top: 2rem; padding-top: 1rem; border-top: 1px solid #21262d; }
88
+ .uncovered-details { border-top: 1px solid #30363d; padding: 1rem; }
89
+ .uncovered-details h3 { color: #f85149; font-size: 0.9rem; margin-bottom: 0.75rem; }
90
+ .uncovered-details ul { list-style: none; }
91
+ .uncovered-details li { font-size: 0.85rem; padding: 0.2rem 0; color: #8b949e; }
92
+ .uncovered-details .operator { color: #c9d1d9; font-family: monospace; margin-right: 0.75rem; }
93
+ .uncovered-details .location { font-family: monospace; }
@@ -7,6 +7,7 @@ require_relative "error_details"
7
7
  require_relative "neutral_details"
8
8
  require_relative "unresolved_details"
9
9
  require_relative "unparseable_details"
10
+ require_relative "uncovered_details"
10
11
 
11
12
  class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HTML::Section
12
13
  template "file_section"
@@ -28,6 +29,10 @@ class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HT
28
29
  @results.count(&:survived?)
29
30
  end
30
31
 
32
+ def uncovered_count
33
+ @results.count(&:uncovered?)
34
+ end
35
+
31
36
  def total
32
37
  @results.length
33
38
  end
@@ -44,6 +49,10 @@ class Evilution::Reporter::HTML::Sections::FileSection < Evilution::Reporter::HT
44
49
  )
45
50
  end
46
51
 
52
+ def uncovered_html
53
+ Evilution::Reporter::HTML::Sections::UncoveredDetails.render_if(@results.select(&:uncovered?))
54
+ end
55
+
47
56
  def error_html
48
57
  Evilution::Reporter::HTML::Sections::ErrorDetails.render_if(@results.select(&:error?))
49
58
  end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../sections"
4
+
5
+ # The mutations no test executes any changed line of: never run, scored as
6
+ # survivors.
7
+ class Evilution::Reporter::HTML::Sections::UncoveredDetails < Evilution::Reporter::HTML::Section
8
+ template "uncovered_details"
9
+
10
+ def self.render_if(uncovered)
11
+ return "" if uncovered.empty?
12
+
13
+ new(uncovered).render
14
+ end
15
+
16
+ def initialize(uncovered)
17
+ @uncovered = uncovered
18
+ end
19
+
20
+ private
21
+
22
+ attr_reader :uncovered
23
+
24
+ def sorted
25
+ uncovered.sort_by { |r| [r.mutation.line, r.mutation.operator_name] }
26
+ end
27
+ end
@@ -1,10 +1,11 @@
1
1
  <section class="file-section">
2
2
  <h2 class="file-header">
3
3
  <span class="file-path"><%= h(@path) %></span>
4
- <span class="file-stats"><%= killed_count %> killed / <%= survived_count %> survived / <%= total %> total</span>
4
+ <span class="file-stats"><%= killed_count %> killed / <%= survived_count %> survived / <%- if uncovered_count.positive? -%><%= uncovered_count %> uncovered / <%- end -%><%= total %> total</span>
5
5
  </h2>
6
6
  <div class="mutation-map"><%= map_html %></div>
7
7
  <%= survived_html %>
8
+ <%= uncovered_html %>
8
9
  <%= error_html %>
9
10
  <%= neutral_html %>
10
11
  <%= unresolved_html %>
@@ -2,6 +2,9 @@
2
2
  <div class="card"><span class="card-value"><%= @summary.total %></span><span class="card-label">Total</span></div>
3
3
  <div class="card card-killed"><span class="card-value"><%= @summary.killed %></span><span class="card-label">Killed</span></div>
4
4
  <div class="card card-survived"><span class="card-value"><%= @summary.survived %></span><span class="card-label">Survived</span></div>
5
+ <%- if @summary.uncovered.positive? -%>
6
+ <div class="card card-survived"><span class="card-value"><%= @summary.uncovered %></span><span class="card-label">Uncovered</span></div>
7
+ <%- end -%>
5
8
  <div class="card"><span class="card-value"><%= @summary.timed_out %></span><span class="card-label">Timed Out</span></div>
6
9
  <div class="card"><span class="card-value"><%= @summary.errors %></span><span class="card-label">Errors</span></div>
7
10
  <div class="card"><span class="card-value"><%= @summary.neutral %></span><span class="card-label">Neutral</span></div>
@@ -0,0 +1,11 @@
1
+ <div class="uncovered-details">
2
+ <h3>Uncovered (<%= uncovered.length %>) — no test executes the lines they change, not run</h3>
3
+ <ul>
4
+ <%- sorted.each do |r| -%>
5
+ <li>
6
+ <span class="operator"><%= h(r.mutation.operator_name) %></span>
7
+ <span class="location"><%= h(r.mutation.file_path) %>:<%= r.mutation.line %></span>
8
+ </li>
9
+ <%- end -%>
10
+ </ul>
11
+ </div>
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../json"
4
+
5
+ # One mutation's entry in a JSON report, whichever bucket it is listed under.
6
+ class Evilution::Reporter::JSON::MutationDetail
7
+ # The mutations that point at a missing test, and get a suggestion for one:
8
+ # those that ran and survived, and those no test executes at all.
9
+ UNTESTED_STATUSES = %i[survived uncovered].freeze
10
+ private_constant :UNTESTED_STATUSES
11
+
12
+ def initialize(suggestion:, timings:)
13
+ @suggestion = suggestion
14
+ @timings = timings
15
+ end
16
+
17
+ def call(result)
18
+ mutation = result.mutation
19
+ detail = base_mutation_fields(mutation, result)
20
+ append_survived_fields(detail, mutation) if UNTESTED_STATUSES.include?(result.status)
21
+ detail[:test_command] = result.test_command if result.test_command
22
+ append_run_fields(detail, result)
23
+ append_memory_fields(detail, result)
24
+ append_error_fields(detail, result)
25
+ append_neutral_reason(detail, result)
26
+ detail.merge!(@timings.mutation_fields(result))
27
+ end
28
+
29
+ private
30
+
31
+ # Why a neutral was recorded — a red baseline and an infrastructure crash want
32
+ # opposite responses, and both land in the same bucket.
33
+ def append_neutral_reason(detail, result)
34
+ reason = result.neutral_reason
35
+ return unless reason
36
+
37
+ detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
38
+ end
39
+
40
+ def base_mutation_fields(mutation, result)
41
+ {
42
+ operator: mutation.operator_name,
43
+ file: mutation.file_path,
44
+ line: mutation.line,
45
+ status: result.status.to_s,
46
+ duration: result.duration.round(4),
47
+ diff: mutation.diff
48
+ }
49
+ end
50
+
51
+ def append_survived_fields(detail, mutation)
52
+ detail[:suggestion] = @suggestion.suggestion_for(mutation)
53
+ unified = mutation.unified_diff
54
+ detail[:unified_diff] = unified if unified
55
+ end
56
+
57
+ # Which test killed the mutation and how many tests its run took, where the
58
+ # integration tells.
59
+ # A survivor reported without a run names the sentinel that implied it.
60
+ def append_run_fields(detail, result)
61
+ detail[:killing_test] = result.killing_test if result.killing_test
62
+ detail[:examples_run] = result.examples_run if result.examples_run
63
+ detail[:implied_by] = result.implied_by if result.implied_by
64
+ end
65
+
66
+ def append_memory_fields(detail, result)
67
+ detail[:parent_rss_kb] = result.parent_rss_kb if result.parent_rss_kb
68
+ detail[:child_rss_kb] = result.child_rss_kb if result.child_rss_kb
69
+ detail[:memory_delta_kb] = result.memory_delta_kb if result.memory_delta_kb
70
+ end
71
+
72
+ def append_error_fields(detail, result)
73
+ detail[:error_message] = result.error_message if result.error_message
74
+ detail[:error_class] = result.error_class if result.error_class
75
+ detail[:error_backtrace] = result.error_backtrace if result.error_backtrace
76
+ end
77
+ end
@@ -22,6 +22,7 @@ class Evilution::Reporter::JSON::Subjects
22
22
  killed: score.killed,
23
23
  verified: score.verified,
24
24
  survived: score.survived,
25
+ uncovered: score.uncovered,
25
26
  score: score.score.round(4),
26
27
  reached: score.reached?
27
28
  }
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../json"
4
+
5
+ # Where the run's time went: each phase of the run, each spec file of the
6
+ # baseline, each mutation phase summed over the mutations, and each worker of
7
+ # a parallel run with the share of its life it spent busy. Every section is
8
+ # always present, empty when the run did not measure it, so a consumer reads
9
+ # one shape.
10
+ class Evilution::Reporter::JSON::Timings
11
+ def call(summary)
12
+ timings = summary.timings
13
+ {
14
+ phases: timings.phases.transform_values { |seconds| seconds.round(4) },
15
+ baseline: timings.baseline_spec_files.map { |file, seconds| { spec_file: file, duration: seconds.round(4) } },
16
+ mutations: summary.mutation_timing_totals.transform_values { |seconds| seconds.round(4) },
17
+ workers: timings.workers.map { |worker| worker_row(worker) }
18
+ }
19
+ end
20
+
21
+ # The fields a mutation's row carries about its own time: where it went,
22
+ # when the run measured it, and the limit it ran under, when it used it up.
23
+ def mutation_fields(result)
24
+ fields = {}
25
+ fields[:time_limit] = result.time_limit.round(4) if result.time_limit
26
+ timings = result.timings
27
+ fields[:timings] = timings.to_h.transform_values { |seconds| seconds.round(4) } if timings
28
+ fields
29
+ end
30
+
31
+ private
32
+
33
+ def worker_row(worker)
34
+ {
35
+ pid: worker.pid,
36
+ items: worker.items_completed,
37
+ busy: worker.busy_time.round(4),
38
+ wall: worker.wall_time.round(4),
39
+ utilization: worker.utilization.round(4)
40
+ }
41
+ end
42
+ end
@@ -8,10 +8,12 @@ require_relative "../reporter"
8
8
  require_relative "../session/schema"
9
9
 
10
10
  class Evilution::Reporter::JSON
11
- def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new, baseline: Baseline.new)
12
- @suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
11
+ def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new, baseline: Baseline.new, timings: Timings.new)
12
+ suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
13
+ @mutation_detail = MutationDetail.new(suggestion: suggestion, timings: timings)
13
14
  @subjects = subjects
14
15
  @baseline = baseline
16
+ @timings = timings
15
17
  end
16
18
 
17
19
  def call(summary)
@@ -28,6 +30,7 @@ class Evilution::Reporter::JSON
28
30
  summary: build_summary(summary),
29
31
  coverage_gaps: build_coverage_gaps(summary),
30
32
  subjects: @subjects.call(summary),
33
+ timings: @timings.call(summary),
31
34
  **result_categories(summary)
32
35
  }
33
36
  append_disabled_to_report(report, summary)
@@ -45,7 +48,8 @@ class Evilution::Reporter::JSON
45
48
  neutral: map_details(summary.neutral_results),
46
49
  equivalent: map_details(summary.equivalent_results),
47
50
  unresolved: map_details(summary.unresolved_results),
48
- unparseable: map_details(summary.unparseable_results)
51
+ unparseable: map_details(summary.unparseable_results),
52
+ uncovered: map_details(summary.uncovered_results)
49
53
  }
50
54
  end
51
55
 
@@ -82,7 +86,8 @@ class Evilution::Reporter::JSON
82
86
  neutral: summary.neutral,
83
87
  equivalent: summary.equivalent,
84
88
  unresolved: summary.unresolved,
85
- unparseable: summary.unparseable
89
+ unparseable: summary.unparseable,
90
+ uncovered: summary.uncovered
86
91
  }
87
92
  end
88
93
 
@@ -100,6 +105,7 @@ class Evilution::Reporter::JSON
100
105
  append_diagnostic_summary_fields(data, summary)
101
106
  data[:truncated] = true if summary.truncated?
102
107
  data[:skipped] = summary.skipped if summary.skipped.positive?
108
+ data[:skipped_sites] = summary.skipped_sites unless summary.skipped_sites.empty?
103
109
  peak = summary.peak_memory_mb
104
110
  data[:peak_memory_mb] = peak.round(1) if peak
105
111
  end
@@ -115,52 +121,7 @@ class Evilution::Reporter::JSON
115
121
  end
116
122
 
117
123
  def build_mutation_detail(result)
118
- mutation = result.mutation
119
- detail = base_mutation_fields(mutation, result)
120
- append_survived_fields(detail, mutation) if result.status == :survived
121
- detail[:test_command] = result.test_command if result.test_command
122
- append_memory_fields(detail, result)
123
- append_error_fields(detail, result)
124
- append_neutral_reason(detail, result)
125
- detail
126
- end
127
-
128
- # Why a neutral was recorded — a red baseline and an infrastructure crash want
129
- # opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
130
- def append_neutral_reason(detail, result)
131
- reason = result.neutral_reason
132
- return unless reason
133
-
134
- detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
135
- end
136
-
137
- def base_mutation_fields(mutation, result)
138
- {
139
- operator: mutation.operator_name,
140
- file: mutation.file_path,
141
- line: mutation.line,
142
- status: result.status.to_s,
143
- duration: result.duration.round(4),
144
- diff: mutation.diff
145
- }
146
- end
147
-
148
- def append_survived_fields(detail, mutation)
149
- detail[:suggestion] = @suggestion.suggestion_for(mutation)
150
- unified = mutation.unified_diff
151
- detail[:unified_diff] = unified if unified
152
- end
153
-
154
- def append_memory_fields(detail, result)
155
- detail[:parent_rss_kb] = result.parent_rss_kb if result.parent_rss_kb
156
- detail[:child_rss_kb] = result.child_rss_kb if result.child_rss_kb
157
- detail[:memory_delta_kb] = result.memory_delta_kb if result.memory_delta_kb
158
- end
159
-
160
- def append_error_fields(detail, result)
161
- detail[:error_message] = result.error_message if result.error_message
162
- detail[:error_class] = result.error_class if result.error_class
163
- detail[:error_backtrace] = result.error_backtrace if result.error_backtrace
124
+ @mutation_detail.call(result)
164
125
  end
165
126
 
166
127
  def build_coverage_gaps(summary)
@@ -188,3 +149,5 @@ end
188
149
 
189
150
  require_relative "json/subjects"
190
151
  require_relative "json/baseline"
152
+ require_relative "json/timings"
153
+ require_relative "json/mutation_detail"
@@ -25,6 +25,11 @@ class Evilution::Result::CoverageGap
25
25
  mutation_results.first.mutation.diff
26
26
  end
27
27
 
28
+ # The survivors a probe-first sentinel implied, which did not run.
29
+ def implied_results
30
+ mutation_results.select(&:implied_by)
31
+ end
32
+
28
33
  def count
29
34
  mutation_results.length
30
35
  end
@@ -3,17 +3,31 @@
3
3
  require_relative "../result"
4
4
  require_relative "error_info"
5
5
  require_relative "memory_stats"
6
+ require_relative "mutation_timings"
6
7
 
7
8
  class Evilution::Result::MutationResult
8
- STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
9
+ STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable uncovered].freeze
9
10
 
10
- attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
11
+ attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason,
12
+ :time_limit, :examples_run, :implied_by
11
13
 
12
14
  # known_failures_only: the tests failed, but every example that did was
13
15
  # already failing before any mutation ran, so the failure says nothing
14
16
  # about this one.
17
+ #
18
+ # time_limit: the seconds the test run was allowed, recorded on a run that
19
+ # used them up so a reader can tell a short adaptive limit from a long one.
20
+ #
21
+ # killing_test: the id of the test whose failure killed the mutation, where
22
+ # the integration names it. examples_run: how many tests ran against the
23
+ # mutation, where the integration counts them.
24
+ #
25
+ # implied_by: the operator of the sentinel mutation whose survival implied
26
+ # this verdict, given without a run (see
27
+ # Runner::MutationExecutor::ProbeFirst); nil for a verdict of its own.
15
28
  def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
16
- test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false)
29
+ test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false,
30
+ timings: nil, time_limit: nil, examples_run: nil, implied_by: nil)
17
31
  raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
18
32
 
19
33
  @mutation = mutation
@@ -25,9 +39,30 @@ class Evilution::Result::MutationResult
25
39
  @error = error
26
40
  @neutral_reason = neutral_reason
27
41
  @known_failures_only = known_failures_only == true
42
+ @timings = timings
43
+ @time_limit = time_limit
44
+ @examples_run = examples_run
45
+ @implied_by = implied_by
28
46
  freeze
29
47
  end
30
48
 
49
+ def with_time_limit(seconds)
50
+ copy(time_limit: seconds)
51
+ end
52
+
53
+ # This result came from running the mutation again after a first run used up
54
+ # a shorter limit; that run is part of what the mutation cost.
55
+ def after_adaptive_timeout(seconds)
56
+ base = timings || Evilution::Result::MutationTimings.new
57
+ copy(duration: @duration + seconds, timings: base.with_adaptive_timeout(seconds))
58
+ end
59
+
60
+ # Where the duration went (Evilution::Result::MutationTimings), when the
61
+ # test process measured it.
62
+ def timings
63
+ @timings.is_a?(Evilution::Result::MutationTimings) ? @timings : nil
64
+ end
65
+
31
66
  # Positive type checks, not nil checks. EV-s5br / GH #1174: the
32
67
  # nil_replacement mutator can swap a nil default into `false` (or some other
33
68
  # non-typed value), and `nil?` then returns false, sending a missing method
@@ -93,4 +128,21 @@ class Evilution::Result::MutationResult
93
128
  def unparseable?
94
129
  status == :unparseable
95
130
  end
131
+
132
+ # No test executes any line the mutation changes, so it was never run: it
133
+ # could only have survived. Scored as a survivor, but kept apart from them.
134
+ def uncovered?
135
+ status == :uncovered
136
+ end
137
+
138
+ private
139
+
140
+ def copy(**changes)
141
+ self.class.new(
142
+ mutation: @mutation, status: @status, duration: @duration, killing_test: @killing_test,
143
+ test_command: @test_command, memory: @memory, error: @error, neutral_reason: @neutral_reason,
144
+ known_failures_only: @known_failures_only, timings: @timings, time_limit: @time_limit, examples_run: @examples_run,
145
+ implied_by: @implied_by, **changes
146
+ )
147
+ end
96
148
  end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../result"
4
+
5
+ # Where one mutation's time went, in seconds:
6
+ #
7
+ # - apply: parsing and evaluating the mutated source
8
+ # - test_run: the first test run against it
9
+ # - confirmation: the re-run of a survivor against its whole spec files
10
+ # - queue_wait: in a parallel run, the time from the dispatcher sending it to a
11
+ # worker to that worker starting it
12
+ # - adaptive_timeout: a first run that used up a limit shorter than the fixed
13
+ # one, after which the mutation was run again under the fixed
14
+ # limit; the other phases then belong to that second run
15
+ #
16
+ # A phase that did not happen is nil. What the mutation's duration holds beyond
17
+ # these is the cost of the isolation around it: fork, sandbox, reap.
18
+ class Evilution::Result::MutationTimings
19
+ PHASES = %i[apply test_run confirmation queue_wait adaptive_timeout].freeze
20
+
21
+ attr_reader(*PHASES)
22
+
23
+ # The timings a test process sent back, which may be missing or not a Hash at
24
+ # all when the mutation broke what produced them.
25
+ def self.from_h(hash)
26
+ return nil unless hash.is_a?(Hash)
27
+
28
+ fields = PHASES.to_h { |phase| [phase, hash[phase]] }.select { |_, value| value.is_a?(Numeric) }
29
+ return nil if fields.empty?
30
+
31
+ new(**fields)
32
+ end
33
+
34
+ def initialize(apply: nil, test_run: nil, confirmation: nil, queue_wait: nil, adaptive_timeout: nil)
35
+ @apply = apply
36
+ @test_run = test_run
37
+ @confirmation = confirmation
38
+ @queue_wait = queue_wait
39
+ @adaptive_timeout = adaptive_timeout
40
+ freeze
41
+ end
42
+
43
+ def with_queue_wait(seconds)
44
+ self.class.new(**to_h, queue_wait: seconds)
45
+ end
46
+
47
+ def with_adaptive_timeout(seconds)
48
+ self.class.new(**to_h, adaptive_timeout: seconds)
49
+ end
50
+
51
+ def to_h
52
+ PHASES.to_h { |phase| [phase, public_send(phase)] }.compact
53
+ end
54
+ end