evilution 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +1 -0
  3. data/.rubocop_todo.yml +30 -0
  4. data/CHANGELOG.md +59 -0
  5. data/README.md +285 -45
  6. data/docs/architecture.md +110 -12
  7. data/docs/ast_pattern_syntax.md +35 -7
  8. data/docs/combined_runs_spike.md +407 -0
  9. data/docs/integrations.md +1 -1
  10. data/docs/isolation.md +63 -2
  11. data/docs/migration-from-mutant.md +16 -11
  12. data/docs/mutant_schemata_spike.md +387 -0
  13. data/docs/operator_yield_study.md +576 -0
  14. data/docs/probe_first_spike.md +264 -0
  15. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  16. data/lib/evilution/ast/callback_declaration.rb +36 -3
  17. data/lib/evilution/ast/class_body.rb +62 -16
  18. data/lib/evilution/ast/def_region.rb +130 -0
  19. data/lib/evilution/ast/local_declaration.rb +37 -0
  20. data/lib/evilution/ast/parent_index.rb +50 -0
  21. data/lib/evilution/ast/parse_cache.rb +27 -0
  22. data/lib/evilution/ast/parser.rb +8 -3
  23. data/lib/evilution/ast/pattern/filter.rb +53 -12
  24. data/lib/evilution/ast/skip_filter.rb +83 -0
  25. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  26. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  27. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  28. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  29. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  30. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  31. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  32. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  33. data/lib/evilution/ast/skip_sites.rb +9 -0
  34. data/lib/evilution/ast/source_surgeon.rb +20 -2
  35. data/lib/evilution/ast/syntax_check.rb +10 -8
  36. data/lib/evilution/baseline/report.rb +11 -2
  37. data/lib/evilution/baseline.rb +31 -6
  38. data/lib/evilution/cache/keys.rb +140 -0
  39. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  40. data/lib/evilution/cache/skeleton.rb +57 -0
  41. data/lib/evilution/cache/spec_digest.rb +39 -0
  42. data/lib/evilution/cache.rb +145 -42
  43. data/lib/evilution/class_tree.rb +38 -0
  44. data/lib/evilution/cli/command.rb +12 -4
  45. data/lib/evilution/cli/commands/run.rb +12 -3
  46. data/lib/evilution/cli/commands/subjects.rb +5 -2
  47. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  48. data/lib/evilution/cli/printers/environment.rb +34 -4
  49. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  50. data/lib/evilution/compare/categorizer.rb +2 -1
  51. data/lib/evilution/compare/normalizer.rb +3 -2
  52. data/lib/evilution/config/sources.rb +3 -0
  53. data/lib/evilution/config/validators/base.rb +9 -0
  54. data/lib/evilution/config/validators/since.rb +22 -0
  55. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  56. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  58. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  59. data/lib/evilution/config.rb +122 -6
  60. data/lib/evilution/coverage/load_tracker.rb +50 -0
  61. data/lib/evilution/coverage/map.rb +42 -5
  62. data/lib/evilution/coverage/map_builder.rb +123 -22
  63. data/lib/evilution/coverage/map_store.rb +15 -5
  64. data/lib/evilution/coverage/recorder.rb +42 -3
  65. data/lib/evilution/coverage_example_filter.rb +35 -0
  66. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  67. data/lib/evilution/example_filter.rb +20 -0
  68. data/lib/evilution/git/changed_lines.rb +103 -0
  69. data/lib/evilution/git/since_scope.rb +77 -0
  70. data/lib/evilution/git/unified_diff.rb +68 -0
  71. data/lib/evilution/infra_crash.rb +26 -0
  72. data/lib/evilution/integration/base.rb +97 -6
  73. data/lib/evilution/integration/crash_detector.rb +46 -6
  74. data/lib/evilution/integration/known_failures.rb +4 -0
  75. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  76. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  77. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  78. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  79. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  80. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  81. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  82. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  83. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  84. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  85. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  86. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  87. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  88. data/lib/evilution/integration/minitest.rb +22 -2
  89. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  90. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  91. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  92. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  93. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  94. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  95. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  96. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  97. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  98. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  99. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  100. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  101. data/lib/evilution/integration/rspec.rb +98 -47
  102. data/lib/evilution/integration/stop_policy.rb +49 -0
  103. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  104. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  105. data/lib/evilution/integration/test_unit.rb +28 -2
  106. data/lib/evilution/isolation/fork.rb +58 -13
  107. data/lib/evilution/isolation/in_process.rb +12 -1
  108. data/lib/evilution/line_set.rb +72 -0
  109. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  110. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  111. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  112. data/lib/evilution/mcp/info_tool.rb +24 -17
  113. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  114. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  115. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  116. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  117. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  118. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  119. data/lib/evilution/mcp/session_tool.rb +1 -2
  120. data/lib/evilution/mcp.rb +4 -0
  121. data/lib/evilution/mutation/line_diff.rb +111 -0
  122. data/lib/evilution/mutation.rb +17 -13
  123. data/lib/evilution/mutator/base.rb +121 -98
  124. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  125. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  126. data/lib/evilution/mutator/load_sources.rb +88 -0
  127. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  128. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  129. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  130. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  131. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  132. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  133. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  135. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  136. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  137. data/lib/evilution/mutator/registry.rb +11 -5
  138. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  139. data/lib/evilution/parallel/pool.rb +11 -4
  140. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  141. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  142. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  143. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  144. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  145. data/lib/evilution/parallel/work_queue.rb +69 -12
  146. data/lib/evilution/parallel_db_warning.rb +39 -12
  147. data/lib/evilution/process_supervisor.rb +22 -2
  148. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  149. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  150. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  151. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  152. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  153. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  154. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  155. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  156. data/lib/evilution/reporter/cli.rb +9 -2
  157. data/lib/evilution/reporter/html/assets/style.css +8 -0
  158. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  159. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  160. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  161. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  162. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  163. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  164. data/lib/evilution/reporter/json/subjects.rb +1 -0
  165. data/lib/evilution/reporter/json/timings.rb +42 -0
  166. data/lib/evilution/reporter/json.rb +13 -50
  167. data/lib/evilution/result/coverage_gap.rb +5 -0
  168. data/lib/evilution/result/mutation_result.rb +55 -3
  169. data/lib/evilution/result/mutation_timings.rb +54 -0
  170. data/lib/evilution/result/run_timings.rb +20 -0
  171. data/lib/evilution/result/subject_score.rb +8 -1
  172. data/lib/evilution/result/subject_scorer.rb +5 -3
  173. data/lib/evilution/result/summary.rb +49 -12
  174. data/lib/evilution/runner/baseline_runner.rb +47 -5
  175. data/lib/evilution/runner/diagnostics.rb +0 -13
  176. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  177. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  178. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  179. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  180. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  181. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  182. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  183. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  184. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  185. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  186. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  187. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  190. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  191. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  195. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  196. data/lib/evilution/runner/mutation_executor.rb +75 -10
  197. data/lib/evilution/runner/mutation_planner.rb +29 -21
  198. data/lib/evilution/runner/phase_clock.rb +22 -0
  199. data/lib/evilution/runner/report_publisher.rb +1 -1
  200. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  201. data/lib/evilution/runner.rb +87 -26
  202. data/lib/evilution/session/diff.rb +9 -2
  203. data/lib/evilution/session/store.rb +17 -2
  204. data/lib/evilution/source_ast_cache.rb +10 -0
  205. data/lib/evilution/spec_ast_cache.rb +15 -0
  206. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  207. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  208. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  209. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  210. data/lib/evilution/spec_resolver.rb +76 -65
  211. data/lib/evilution/spec_selector.rb +17 -13
  212. data/lib/evilution/subject.rb +7 -2
  213. data/lib/evilution/version.rb +1 -1
  214. data/lib/evilution/worker_database/copies.rb +73 -0
  215. data/lib/evilution/worker_database/schema.rb +39 -0
  216. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  217. data/lib/evilution/worker_database.rb +214 -0
  218. data/schema/evilution.config.schema.json +74 -2
  219. metadata +84 -13
  220. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  221. data/scripts/benchmark_density +0 -262
  222. data/scripts/benchmark_density.yml +0 -19
  223. data/scripts/canary_manifest.yml +0 -47
  224. data/scripts/compare_mutations +0 -421
  225. data/scripts/compare_mutations.yml +0 -24
  226. data/scripts/compare_targeting +0 -279
  227. data/scripts/compare_targeting.example.yml +0 -24
  228. data/scripts/mutant_json_adapter +0 -227
@@ -1,279 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # frozen_string_literal: true
3
-
4
- # Targeting-mode comparison harness.
5
- #
6
- # For each repo in a manifest, runs evilution over the SAME mutation set under
7
- # three example-targeting modes -- full_file (baseline), lexical (current), and
8
- # coverage (new) -- joins the per-mutation results, and emits a per-repo table:
9
- # score_full / score_lexical / score_coverage,
10
- # lost_kills (mutations full_file KILLED but coverage did NOT -> the gate),
11
- # wall_ratio_lexical / wall_ratio_coverage (vs the full_file baseline).
12
- #
13
- # The accuracy gate is lost_kills == 0: coverage targeting must never lose a kill
14
- # the full-file run would catch. Speed is reported, not gated.
15
- #
16
- # Usage:
17
- # scripts/compare_targeting MANIFEST.yml [--out report.md]
18
- #
19
- # MANIFEST.yml:
20
- # repos:
21
- # - name: acme/foo
22
- # dir: /checkouts/foo # already bundled
23
- # args: ["lib/foo.rb", "--jobs", "4"]
24
- #
25
- # The actual canary execution (checkout + bundle over the EV-rxob manifest) is
26
- # the validation run, EV-7ydn; this harness is the reusable comparison engine.
27
-
28
- require "json"
29
- require "yaml"
30
- require "open3"
31
- require "optparse"
32
- require "digest"
33
-
34
- module CompareTargeting
35
- MODES = %w[full_file lexical coverage].freeze
36
- KEY_FIELDS = %w[file line operator].freeze
37
-
38
- class ConfigError < StandardError; end
39
-
40
- module_function
41
-
42
- # Stable per-mutation identity across modes: only TARGETING differs between
43
- # runs, so the same mutation has the same file/line/operator (+ diff to
44
- # separate distinct mutations sharing a line+operator).
45
- def key_for(detail)
46
- base = KEY_FIELDS.map { |field| detail[field] }.join(":")
47
- digest = detail["diff"].to_s
48
- digest.empty? ? base : "#{base}##{Digest::SHA256.hexdigest(digest)[0, 8]}"
49
- end
50
-
51
- # The label shown in lost-kill output: file:line:operator (no diff hash).
52
- def label_for(detail)
53
- KEY_FIELDS.map { |field| detail[field] }.join(":")
54
- end
55
-
56
- # One mode's run, indexed by mutation key.
57
- class ModeResult
58
- CATEGORIES = %w[killed survived neutral equivalent unresolved unparseable timed_out errors].freeze
59
-
60
- def self.from_json(data)
61
- by_key = {}
62
- CATEGORIES.each do |category|
63
- Array(data[category]).each do |detail|
64
- key = CompareTargeting.key_for(detail)
65
- by_key[key] ||= {
66
- status: detail["status"],
67
- duration: (detail["duration"] || 0.0).to_f,
68
- label: CompareTargeting.label_for(detail)
69
- }
70
- end
71
- end
72
- new(by_key)
73
- end
74
-
75
- def initialize(by_key)
76
- @by_key = by_key
77
- end
78
-
79
- def keys
80
- @by_key.keys
81
- end
82
-
83
- def status(key)
84
- @by_key.dig(key, :status)
85
- end
86
-
87
- def label(key)
88
- @by_key.dig(key, :label)
89
- end
90
-
91
- def killed_count
92
- @by_key.count { |_, value| value[:status] == "killed" }
93
- end
94
-
95
- # Measurable = the run actually observed a verdict: killed or survived.
96
- # unresolved/equivalent/errors/unparseable are excluded from the score.
97
- def measurable_count
98
- @by_key.count { |_, value| %w[killed survived].include?(value[:status]) }
99
- end
100
-
101
- def total_duration
102
- @by_key.values.sum { |value| value[:duration] }
103
- end
104
- end
105
-
106
- # Joins the three modes for one repo and derives the comparison metrics.
107
- class Comparison
108
- def initialize(full_file:, lexical:, coverage:)
109
- @modes = { "full_file" => full_file, "lexical" => lexical, "coverage" => coverage }
110
- end
111
-
112
- def score(mode)
113
- result = @modes.fetch(mode)
114
- return 0.0 if result.measurable_count.zero?
115
-
116
- result.killed_count.to_f / result.measurable_count
117
- end
118
-
119
- def wall_ratio(mode)
120
- baseline = @modes.fetch("full_file").total_duration
121
- return 0.0 if baseline.zero?
122
-
123
- @modes.fetch(mode).total_duration / baseline
124
- end
125
-
126
- # Mutation labels that full_file KILLED but `mode` failed to kill -- the
127
- # lost kills that must be zero before coverage can become the default.
128
- def lost_kills(mode = "coverage")
129
- full = @modes.fetch("full_file")
130
- other = @modes.fetch(mode)
131
- full
132
- .keys
133
- .select { |key| full.status(key) == "killed" && other.status(key) != "killed" }
134
- .map { |key| full.label(key) }
135
- end
136
-
137
- def to_row(repo)
138
- {
139
- repo: repo,
140
- score_full: score("full_file"),
141
- score_lexical: score("lexical"),
142
- score_coverage: score("coverage"),
143
- lost_kills: lost_kills("coverage").size,
144
- wall_ratio_lexical: wall_ratio("lexical"),
145
- wall_ratio_coverage: wall_ratio("coverage")
146
- }
147
- end
148
- end
149
-
150
- # Renders the per-repo rows as a markdown table with a PASS/FAIL gate line.
151
- class TableReporter
152
- COLUMNS = %w[repo score_full score_lexical score_coverage lost_kills
153
- wall_ratio_lexical wall_ratio_coverage].freeze
154
-
155
- def initialize(rows)
156
- @rows = rows
157
- end
158
-
159
- def to_markdown
160
- (table_lines + ["", gate_line]).join("\n")
161
- end
162
-
163
- private
164
-
165
- def table_lines
166
- [header_row, separator_row, *@rows.map { |row| data_row(row) }]
167
- end
168
-
169
- def header_row
170
- "| #{COLUMNS.join(" | ")} |"
171
- end
172
-
173
- def separator_row
174
- "| #{COLUMNS.map { "---" }.join(" | ")} |"
175
- end
176
-
177
- def data_row(row)
178
- "| #{COLUMNS.map { |col| format_cell(row[col.to_sym]) }.join(" | ")} |"
179
- end
180
-
181
- def gate_line
182
- total_lost = @rows.sum { |row| row[:lost_kills] }
183
- gate = total_lost.zero? ? "PASS" : "FAIL"
184
- "GATE (total lost_kills == 0): #{gate} (#{total_lost} lost kills across #{@rows.size} repos)"
185
- end
186
-
187
- def format_cell(value)
188
- value.is_a?(Float) ? format("%.3f", value) : value.to_s
189
- end
190
- end
191
-
192
- # Runs evilution for one (repo, mode) and parses the JSON into a ModeResult.
193
- # command_runner is injected so the pure pipeline is testable without a real
194
- # mutation run.
195
- class ModeRunner
196
- # Run the inner `bundle exec evilution` in the TARGET repo's bundler context.
197
- # If this harness is itself launched under `bundle exec`, Bundler exports
198
- # BUNDLE_GEMFILE/RUBYOPT into the child, which would make the inner bundle
199
- # resolve against evilution's Gemfile instead of the target repo's. Strip
200
- # that inherited bundler env first.
201
- DEFAULT_RUNNER = lambda do |cmd, dir|
202
- stdout, stderr, status =
203
- if defined?(Bundler)
204
- Bundler.with_unbundled_env { Open3.capture3(*cmd, chdir: dir) }
205
- else
206
- Open3.capture3(*cmd, chdir: dir)
207
- end
208
- raise ConfigError, "evilution failed in #{dir}: #{stderr}" unless status.success?
209
-
210
- stdout
211
- end
212
-
213
- def initialize(command_runner: DEFAULT_RUNNER)
214
- @command_runner = command_runner
215
- end
216
-
217
- def run(repo_dir:, evilution_args:, mode:)
218
- cmd = ["bundle", "exec", "evilution", *evilution_args,
219
- "--example-targeting", mode, "--format", "json"]
220
- json = @command_runner.call(cmd, repo_dir)
221
- ModeResult.from_json(JSON.parse(json))
222
- end
223
- end
224
-
225
- # Drives the manifest: every repo through every mode, then the table.
226
- class Harness
227
- def initialize(mode_runner: ModeRunner.new)
228
- @mode_runner = mode_runner
229
- end
230
-
231
- def call(repos)
232
- rows = repos.map { |repo| compare_repo(repo) }
233
- TableReporter.new(rows).to_markdown
234
- end
235
-
236
- def compare_repo(repo)
237
- results = MODES.to_h do |mode|
238
- [mode, @mode_runner.run(repo_dir: repo.fetch(:dir), evilution_args: repo.fetch(:args), mode: mode)]
239
- end
240
- Comparison.new(**results.transform_keys(&:to_sym)).to_row(repo.fetch(:name))
241
- end
242
- end
243
-
244
- def load_manifest(path)
245
- data = YAML.safe_load_file(path, symbolize_names: true)
246
- raise ConfigError, "manifest must list repos:" unless data.is_a?(Hash) && data[:repos].is_a?(Array)
247
-
248
- data[:repos]
249
- rescue Errno::ENOENT
250
- raise ConfigError, "manifest not found: #{path}"
251
- rescue Psych::SyntaxError => e
252
- raise ConfigError, "manifest is not valid YAML: #{e.message}"
253
- end
254
- end
255
-
256
- if __FILE__ == $PROGRAM_NAME
257
- out_path = nil
258
- parser = OptionParser.new do |opts|
259
- opts.banner = "Usage: scripts/compare_targeting MANIFEST.yml [--out report.md]"
260
- opts.on("--out PATH", "Write the markdown report to PATH (default: stdout)") { |p| out_path = p }
261
- end
262
- parser.parse!
263
-
264
- manifest_path = ARGV.first
265
- unless manifest_path
266
- warn parser.banner
267
- exit 2
268
- end
269
-
270
- begin
271
- repos = CompareTargeting.load_manifest(manifest_path)
272
- report = CompareTargeting::Harness.new.call(repos)
273
- out_path ? File.write(out_path, report) : puts(report)
274
- exit 0
275
- rescue CompareTargeting::ConfigError => e
276
- warn "Error: #{e.message}"
277
- exit 2
278
- end
279
- end
@@ -1,24 +0,0 @@
1
- # Example manifest for scripts/compare_targeting (EV-51d4).
2
- #
3
- # Each repo is run through evilution three times -- once per example-targeting
4
- # mode (full_file, lexical, coverage) -- over the SAME mutation set, and the
5
- # harness emits a per-repo table with the lost_kills gate and wall-time ratios.
6
- #
7
- # Repos must already be checked out and `bundle install`ed; the harness only
8
- # runs evilution inside them. The real EV-rxob canary list is wired up in the
9
- # validation run (EV-7ydn).
10
- #
11
- # scripts/compare_targeting scripts/compare_targeting.example.yml --out report.md
12
-
13
- repos:
14
- - name: thoughtbot/factory_bot
15
- dir: /tmp/ev-canaries/factory_bot
16
- args: ["lib", "--jobs", "4"]
17
-
18
- - name: bblimke/webmock
19
- dir: /tmp/ev-canaries/webmock
20
- args: ["lib", "--jobs", "4"]
21
-
22
- - name: rack/rack
23
- dir: /tmp/ev-canaries/rack
24
- args: ["lib", "--jobs", "4"]
@@ -1,227 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # frozen_string_literal: true
3
-
4
- # Mutant JSON Adapter
5
- #
6
- # Extracts method bodies from a Ruby file using Prism, feeds each to
7
- # `mutant util mutation -e`, parses the diff output, and produces JSON
8
- # in the same schema as `evilution util mutation --format json`:
9
- #
10
- # [{"operator": "...", "subject": "...", "file": "...", "line": N, "diff": "..."}, ...]
11
- #
12
- # This works around mutant's inability to parse Ruby 4.0 files by processing
13
- # individual method snippets through its eval mode.
14
- #
15
- # Usage:
16
- # scripts/mutant_json_adapter FILE_PATH [--project-root DIR] [--verbose]
17
-
18
- require "json"
19
- require "open3"
20
- require "optparse"
21
- require "prism"
22
-
23
- module MutantJsonAdapter
24
- class ParseError < StandardError; end
25
-
26
- class MethodExtractor
27
- MethodInfo = Struct.new(:name, :line_number, :source)
28
-
29
- def initialize(file_path)
30
- @file_path = file_path
31
- end
32
-
33
- def extract
34
- source = File.read(@file_path)
35
- binary = source.b
36
- encoding = source.encoding
37
- result = Prism.parse(source)
38
-
39
- raise ParseError, "failed to parse #{@file_path}" if result.failure?
40
-
41
- visitor = MethodVisitor.new(binary, encoding)
42
- visitor.visit(result.value)
43
- visitor.methods
44
- end
45
-
46
- class MethodVisitor < Prism::Visitor
47
- attr_reader :methods
48
-
49
- def initialize(source_binary, encoding)
50
- super()
51
- @source_binary = source_binary
52
- @encoding = encoding
53
- @methods = []
54
- @context = []
55
- end
56
-
57
- def visit_module_node(node)
58
- @context.push(node.constant_path&.full_name || node.name.to_s)
59
- super
60
- @context.pop
61
- end
62
-
63
- def visit_class_node(node)
64
- @context.push(node.constant_path&.full_name || node.name.to_s)
65
- super
66
- @context.pop
67
- end
68
-
69
- def visit_def_node(node)
70
- loc = node.location
71
- text = @source_binary[loc.start_offset...loc.end_offset].force_encoding(@encoding)
72
- separator = node.receiver ? "." : "#"
73
- name = "#{@context.join("::")}#{separator}#{node.name}"
74
-
75
- @methods << MethodInfo.new(name: name, line_number: loc.start_line, source: text)
76
- super
77
- end
78
- end
79
- end
80
-
81
- class MutantRunner
82
- def initialize(project_root: nil)
83
- @project_root = project_root
84
- end
85
-
86
- def mutations_for(source)
87
- cmd = ["bundle", "exec", "mutant", "util", "mutation", "-e", source]
88
- opts = @project_root ? { chdir: @project_root } : {}
89
- stdout, _stderr, _status = Open3.capture3(*cmd, **opts)
90
- stdout
91
- end
92
- end
93
-
94
- class OutputParser
95
- def parse(raw_output, subject:, file_path:)
96
- return [] if raw_output.strip.empty?
97
-
98
- mutations = []
99
- current_diff_lines = []
100
-
101
- raw_output.each_line do |line|
102
- if line.start_with?("@@ ")
103
- flush_mutation(mutations, current_diff_lines, subject, file_path) if current_diff_lines.any?
104
- current_diff_lines = [line]
105
- elsif line.start_with?("-", "+", " ") && current_diff_lines.any?
106
- current_diff_lines << line
107
- end
108
- end
109
-
110
- flush_mutation(mutations, current_diff_lines, subject, file_path) if current_diff_lines.any?
111
- mutations
112
- end
113
-
114
- private
115
-
116
- def flush_mutation(mutations, diff_lines, subject, file_path)
117
- diff_text = diff_lines.join
118
- operator = infer_operator(diff_lines)
119
-
120
- mutations << {
121
- "operator" => operator,
122
- "subject" => subject.name,
123
- "file" => file_path,
124
- "line" => subject.line_number,
125
- "diff" => diff_text
126
- }
127
- diff_lines.clear
128
- end
129
-
130
- def infer_operator(diff_lines)
131
- removed = extract_diff_side(diff_lines, "-", "---")
132
- added = extract_diff_side(diff_lines, "+", "+++")
133
-
134
- categorize_mutation(removed, added)
135
- end
136
-
137
- def extract_diff_side(diff_lines, prefix, header)
138
- diff_lines.select { |l| l.start_with?(prefix) && !l.start_with?(header) }
139
- .map { |l| l[1..].strip }
140
- end
141
-
142
- def categorize_mutation(removed, added)
143
- return "replacement" if removed.empty?
144
-
145
- body_lines = added.reject { |l| l == "end" || l.start_with?("def ") }
146
- return "removal" if body_lines.empty?
147
-
148
- classify_added(body_lines.first)
149
- end
150
-
151
- def classify_added(added_body)
152
- case added_body
153
- when "raise" then "raise_insertion"
154
- when "super" then "super_replacement"
155
- when "nil" then "nil_replacement"
156
- when "true", "false" then "boolean_swap"
157
- when /\.eql\?/ then "equality_change"
158
- when /\.equal\?/ then "identity_change"
159
- when /-?\d+\b/ then "literal_boundary"
160
- else "mutation"
161
- end
162
- end
163
- end
164
-
165
- class Runner
166
- def initialize(file_path:, project_root: nil, verbose: false)
167
- @file_path = File.expand_path(file_path)
168
- @project_root = project_root
169
- @verbose = verbose
170
- end
171
-
172
- def call
173
- subjects = MethodExtractor.new(@file_path).extract
174
-
175
- if subjects.empty?
176
- warn "No methods found in #{@file_path}" if @verbose
177
- puts "[]"
178
- return 0
179
- end
180
-
181
- all_mutations = process_subjects(subjects)
182
- puts JSON.pretty_generate(all_mutations)
183
- 0
184
- rescue ParseError => e
185
- warn e.message
186
- 2
187
- end
188
-
189
- private
190
-
191
- def process_subjects(subjects)
192
- mutant_runner = MutantRunner.new(project_root: @project_root)
193
- parser = OutputParser.new
194
-
195
- subjects.flat_map do |subject|
196
- warn " Processing #{subject.name}..." if @verbose
197
- raw = mutant_runner.mutations_for(subject.source)
198
- parser.parse(raw, subject: subject, file_path: @file_path)
199
- end
200
- end
201
- end
202
- end
203
-
204
- if __FILE__ == $PROGRAM_NAME
205
- verbose = false
206
- project_root = nil
207
-
208
- opt_parser = OptionParser.new do |opts|
209
- opts.banner = "Usage: #{$PROGRAM_NAME} FILE_PATH [--project-root DIR] [--verbose]"
210
- opts.on("-v", "--verbose", "Show progress") { verbose = true }
211
- opts.on("-p", "--project-root DIR", "Project root for bundle exec") { |d| project_root = d }
212
- opts.on("-h", "--help", "Show help") do
213
- puts opts
214
- exit 0
215
- end
216
- end
217
- opt_parser.parse!
218
-
219
- file_path = ARGV.first
220
- unless file_path
221
- warn opt_parser.banner
222
- exit 2
223
- end
224
-
225
- runner = MutantJsonAdapter::Runner.new(file_path: file_path, project_root: project_root, verbose: verbose)
226
- exit runner.call
227
- end