evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "../state_guard"
4
- require_relative "../../rspec"
5
-
6
- class Evilution::Integration::RSpec::StateGuard::ObjectSpaceExampleGroups
7
- def snapshot
8
- groups = Set.new
9
- ObjectSpace.each_object(Class) do |klass|
10
- groups << klass.object_id if klass < ::RSpec::Core::ExampleGroup
11
- rescue TypeError
12
- # ObjectSpace iteration may surface partially-initialized or anonymous
13
- # classes whose `<` comparison raises. Skipping them is safe — they
14
- # cannot be ExampleGroup descendants we need to track.
15
- end
16
- groups
17
- end
18
-
19
- def release(eg_before)
20
- return unless eg_before
21
-
22
- ObjectSpace.each_object(Class) do |klass|
23
- next unless klass < ::RSpec::Core::ExampleGroup
24
- next if eg_before.include?(klass.object_id)
25
-
26
- klass.constants(false).each do |const|
27
- klass.send(:remove_const, const)
28
- rescue NameError
29
- # Constant may have been removed concurrently (e.g. via autoload
30
- # reload) between #constants(false) and #remove_const. Best-effort
31
- # cleanup — nothing to do if it's already gone.
32
- end
33
-
34
- klass.instance_variables.each do |ivar|
35
- klass.remove_instance_variable(ivar)
36
- end
37
- rescue TypeError
38
- # Same defensive case as #snapshot: skip classes whose `<` raises
39
- # mid-iteration.
40
- end
41
- end
42
- end
@@ -1,262 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # frozen_string_literal: true
3
-
4
- # Mutation Density Benchmark
5
- #
6
- # Compares mutation counts between evilution and a reference mutation testing
7
- # tool on a corpus of files.
8
- # See docs/mutation_density_benchmark.md for methodology.
9
- #
10
- # Usage:
11
- # scripts/benchmark_density CONFIG_FILE [--verbose]
12
- #
13
- # The config file is a YAML file listing the benchmark corpus.
14
- # See scripts/benchmark_density.yml for the format.
15
-
16
- require "yaml"
17
- require "open3"
18
- require "optparse"
19
-
20
- module BenchmarkDensity
21
- class ConfigError < StandardError; end
22
-
23
- class Config
24
- attr_reader :project_root, :target_ratio, :reference_cmd, :files
25
-
26
- def initialize(path)
27
- data = YAML.safe_load_file(path, symbolize_names: true, permitted_classes: [Symbol])
28
- raise ConfigError, "config file must contain a YAML mapping" unless data.is_a?(Hash)
29
-
30
- @project_root = data[:project_root]
31
- @target_ratio = data.key?(:target_ratio) ? data[:target_ratio] : 1.5
32
- @reference_cmd = data.key?(:reference_cmd) ? data[:reference_cmd] : []
33
- @files = data.key?(:files) ? data[:files] : []
34
-
35
- validate!
36
- rescue Psych::Exception => e
37
- raise ConfigError, "failed to parse config file #{path}: #{e.message}"
38
- end
39
-
40
- private
41
-
42
- def validate!
43
- raise ConfigError, "project_root is required in config" if @project_root.nil?
44
- raise ConfigError, "project_root does not exist: #{@project_root}" unless Dir.exist?(@project_root)
45
- raise ConfigError, "reference_cmd is required in config" if @reference_cmd.empty?
46
- raise ConfigError, "files list is empty — add benchmark files to the config" if @files.empty?
47
-
48
- @files.each_with_index { |entry, index| validate_file_entry!(entry, index) }
49
- end
50
-
51
- def validate_file_entry!(entry, index)
52
- raise ConfigError, "files[#{index}] must be a mapping" unless entry.is_a?(Hash)
53
-
54
- path = entry[:path]
55
- target = entry[:reference_target]
56
- raise ConfigError, "files[#{index}] is missing required path" if path.nil? || path.to_s.strip.empty?
57
- raise ConfigError, "files[#{index}] is missing required reference_target" if target.nil? || target.to_s.strip.empty?
58
- end
59
- end
60
-
61
- class EvilutionCounter
62
- def initialize(project_root)
63
- @project_root = project_root
64
- end
65
-
66
- def count_mutations(file_path)
67
- full_path = File.join(@project_root, file_path)
68
- cmd = ["bundle", "exec", "evilution", "subjects", full_path]
69
- stdout, stderr, status = Open3.capture3(*cmd, chdir: @project_root)
70
-
71
- unless status.success?
72
- warn " evilution failed for #{file_path}: #{stderr.strip}"
73
- return nil
74
- end
75
-
76
- count = parse_subject_count(stdout)
77
- if count.nil?
78
- warn " evilution produced unparseable output for #{file_path}: #{stdout.lines.first&.strip}"
79
- return nil
80
- end
81
- count
82
- end
83
-
84
- private
85
-
86
- def parse_subject_count(output)
87
- match = output.match(/(\d+)\s+subjects,\s+(\d+)\s+mutations/)
88
- match ? match[2].to_i : nil
89
- end
90
- end
91
-
92
- class ReferenceCounter
93
- def initialize(project_root, reference_cmd)
94
- @project_root = project_root
95
- @reference_cmd = reference_cmd
96
- end
97
-
98
- def count_mutations(reference_target)
99
- cmd = @reference_cmd + [reference_target]
100
- stdout, stderr, _status = Open3.capture3(*cmd, chdir: @project_root)
101
-
102
- # Reference tool may exit non-zero when mutants survive, which is expected.
103
- # Only treat it as failure if there's no parseable output.
104
- count = parse_mutation_count(stdout)
105
- if count.nil?
106
- warn " reference tool failed for #{reference_target}: #{stderr.lines.first&.strip}"
107
- return nil
108
- end
109
- count
110
- end
111
-
112
- private
113
-
114
- def parse_mutation_count(output)
115
- match = output.match(/mutations:\s*(\d+)/i)
116
- match ? match[1].to_i : nil
117
- end
118
- end
119
-
120
- class Runner
121
- HEADER_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
122
- ROW_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
123
- TOTAL_FMT = "%-45<file>s %10<evilution>d %10<reference>d %7.2<ratio>fx"
124
-
125
- def initialize(config:, verbose: false)
126
- @config = config
127
- @verbose = verbose
128
- @evilution = EvilutionCounter.new(config.project_root)
129
- @reference = ReferenceCounter.new(config.project_root, config.reference_cmd)
130
- end
131
-
132
- def call
133
- results = collect_results
134
- print_table(results)
135
- print_summary(results)
136
- passing?(results) ? 0 : 1
137
- end
138
-
139
- private
140
-
141
- def collect_results
142
- @config.files.map do |entry|
143
- path = entry[:path]
144
- reference_target = entry[:reference_target]
145
-
146
- warn " Measuring #{path}..." if @verbose
147
- ev_count = @evilution.count_mutations(path)
148
- ref_count = @reference.count_mutations(reference_target)
149
- warn " done" if @verbose
150
-
151
- { path: path, evilution: ev_count, reference: ref_count }
152
- end
153
- end
154
-
155
- def print_table(results)
156
- puts format(HEADER_FMT, file: "File", evilution: "Evilution", reference: "Reference", ratio: "Ratio")
157
- puts "-" * 75
158
- results.each { |r| puts format_result_row(r) }
159
- puts "-" * 75
160
- end
161
-
162
- def format_result_row(result)
163
- ratio = compute_ratio(result[:evilution], result[:reference])
164
- format(ROW_FMT,
165
- file: result[:path],
166
- evilution: result[:evilution]&.to_s || "ERR",
167
- reference: result[:reference]&.to_s || "ERR",
168
- ratio: ratio ? format("%.2fx", ratio) : "N/A")
169
- end
170
-
171
- def print_summary(results)
172
- totals = compute_totals(results)
173
- print_total_line(totals)
174
- puts ""
175
- print_status_line(totals)
176
- print_error_warning(totals[:error_count])
177
- end
178
-
179
- def print_total_line(totals)
180
- if totals[:invalid]
181
- puts format(ROW_FMT,
182
- file: "TOTAL", evilution: totals[:ev].to_s, reference: totals[:ref].to_s, ratio: "N/A")
183
- else
184
- puts format(TOTAL_FMT,
185
- file: "TOTAL", evilution: totals[:ev], reference: totals[:ref], ratio: totals[:display_ratio])
186
- end
187
- end
188
-
189
- def compute_totals(results)
190
- valid, errored = partition_results(results)
191
- ev_total = valid.sum { |r| r[:evilution] }
192
- ref_total = valid.sum { |r| r[:reference] }
193
- invalid = errored.any? || ev_total.zero?
194
- ratio = ev_total.positive? ? ref_total.to_f / ev_total : 0.0
195
-
196
- { ev: ev_total, ref: ref_total, ratio: ratio, display_ratio: ratio.round(2),
197
- invalid: invalid, error_count: errored.length }
198
- end
199
-
200
- def partition_results(results)
201
- valid = results.select { |r| r[:evilution] && r[:reference] }
202
- errored = results - valid
203
- [valid, errored]
204
- end
205
-
206
- def print_status_line(totals)
207
- target = @config.target_ratio
208
- if totals[:invalid]
209
- puts "Density ratio: INVALID (target: < #{target}x) — FAIL"
210
- return
211
- end
212
-
213
- status = totals[:ratio] < target ? "PASS" : "FAIL"
214
- puts "Density ratio: #{totals[:display_ratio]}x (target: < #{target}x) — #{status}"
215
- end
216
-
217
- def print_error_warning(error_count)
218
- puts "WARNING: #{error_count} file(s) had errors — benchmark marked invalid" if error_count.positive?
219
- end
220
-
221
- def passing?(results)
222
- totals = compute_totals(results)
223
- return false if totals[:invalid]
224
-
225
- totals[:ratio] < @config.target_ratio
226
- end
227
-
228
- def compute_ratio(evilution, reference)
229
- return nil if evilution.nil? || reference.nil? || evilution.zero?
230
-
231
- (reference.to_f / evilution).round(2)
232
- end
233
- end
234
- end
235
-
236
- if __FILE__ == $PROGRAM_NAME
237
- verbose = false
238
- parser = OptionParser.new do |opts|
239
- opts.banner = "Usage: #{$PROGRAM_NAME} CONFIG_FILE [--verbose]"
240
- opts.on("-v", "--verbose", "Show progress") { verbose = true }
241
- opts.on("-h", "--help", "Show help") do
242
- puts opts
243
- exit 0
244
- end
245
- end
246
- parser.parse!
247
-
248
- config_path = ARGV.first
249
- unless config_path
250
- warn parser.banner
251
- exit 2
252
- end
253
-
254
- begin
255
- config = BenchmarkDensity::Config.new(config_path)
256
- runner = BenchmarkDensity::Runner.new(config: config, verbose: verbose)
257
- exit runner.call
258
- rescue BenchmarkDensity::ConfigError => e
259
- warn "Config error: #{e.message}"
260
- exit 2
261
- end
262
- end
@@ -1,19 +0,0 @@
1
- # Benchmark corpus configuration
2
- #
3
- # See docs/mutation_density_benchmark.md for selection criteria and methodology.
4
- #
5
- # Example:
6
- #
7
- # project_root: /path/to/rails/app
8
- # target_ratio: 1.5
9
- # reference_cmd: ["bundle", "exec", "mutant", "run", "--integration", "rspec", "--fail-fast"]
10
- # files:
11
- # - path: app/models/user.rb
12
- # reference_target: "User"
13
- # - path: app/services/payment_processor.rb
14
- # reference_target: "PaymentProcessor"
15
-
16
- project_root: null # Set to the absolute path of the target project
17
- target_ratio: 1.5
18
- reference_cmd: [] # Command prefix for the reference tool
19
- files: []
@@ -1,47 +0,0 @@
1
- # Canary manifest for the EV-7ydn validation run of scripts/compare_targeting.
2
- #
3
- # scripts/compare_targeting scripts/canary_manifest.yml --out .artifacts/targeting_validation.md
4
- #
5
- # Each repo is run through evilution three times (full_file / lexical / coverage)
6
- # over the SAME mutation set; the gate is total lost_kills == 0 (coverage must
7
- # never lose a kill full-file caught) before the default flips to coverage.
8
- #
9
- # `dir:` assumes the EV-rxob canary checkout layout (/tmp/ev-canaries/<repo>,
10
- # already `bundle install`ed). Adjust to your checkout. Per-repo `args` mirror
11
- # the EV-rxob R2 findings: repos whose specs are NOT lib-mirrored need an
12
- # explicit --spec (EV-z7f5 / GH #1325), or auto spec-resolution yields 0.0 and
13
- # the comparison is meaningless.
14
- #
15
- # Start with the R2 infra=none tier that ran clean; extend with the DB-tier
16
- # repos once their services are up.
17
-
18
- repos:
19
- # --- R2 PASS, lib-mirrored specs (auto-resolution works) ---
20
- - name: thoughtbot/factory_bot
21
- dir: /tmp/ev-canaries/factory_bot
22
- args: ["lib", "--jobs", "4"]
23
-
24
- - name: jnunemaker/httparty
25
- dir: /tmp/ev-canaries/httparty
26
- args: ["lib", "--jobs", "4"]
27
-
28
- - name: rubocop/rubocop
29
- dir: /tmp/ev-canaries/rubocop
30
- args: ["lib", "--jobs", "4"]
31
-
32
- - name: rack/rack
33
- dir: /tmp/ev-canaries/rack
34
- args: ["lib", "--jobs", "4"]
35
-
36
- # --- Non-lib-mirrored specs: explicit --spec required (EV-z7f5 / GH #1325) ---
37
- - name: bblimke/webmock
38
- dir: /tmp/ev-canaries/webmock
39
- args: ["lib", "--spec", "spec/unit", "--jobs", "4"]
40
-
41
- - name: doorkeeper-gem/doorkeeper
42
- dir: /tmp/ev-canaries/doorkeeper
43
- args: ["lib", "--spec", "spec", "--jobs", "4"]
44
-
45
- - name: ruby-concurrency/concurrent-ruby
46
- dir: /tmp/ev-canaries/concurrent-ruby
47
- args: ["lib", "--spec", "spec", "--jobs", "4"]