evilution 1.5.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +30 -0
  3. data/CHANGELOG.md +49 -0
  4. data/README.md +285 -45
  5. data/docs/architecture.md +110 -12
  6. data/docs/ast_pattern_syntax.md +35 -7
  7. data/docs/combined_runs_spike.md +407 -0
  8. data/docs/integrations.md +1 -1
  9. data/docs/isolation.md +63 -2
  10. data/docs/migration-from-mutant.md +16 -11
  11. data/docs/mutant_schemata_spike.md +387 -0
  12. data/docs/operator_yield_study.md +576 -0
  13. data/docs/probe_first_spike.md +264 -0
  14. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  15. data/lib/evilution/ast/callback_declaration.rb +36 -3
  16. data/lib/evilution/ast/class_body.rb +62 -16
  17. data/lib/evilution/ast/def_region.rb +130 -0
  18. data/lib/evilution/ast/local_declaration.rb +37 -0
  19. data/lib/evilution/ast/parent_index.rb +50 -0
  20. data/lib/evilution/ast/parse_cache.rb +27 -0
  21. data/lib/evilution/ast/parser.rb +8 -3
  22. data/lib/evilution/ast/pattern/filter.rb +53 -12
  23. data/lib/evilution/ast/skip_filter.rb +83 -0
  24. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  25. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  26. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  27. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  28. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  29. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  30. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  31. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  32. data/lib/evilution/ast/skip_sites.rb +9 -0
  33. data/lib/evilution/ast/source_surgeon.rb +20 -2
  34. data/lib/evilution/ast/syntax_check.rb +10 -8
  35. data/lib/evilution/baseline/report.rb +11 -2
  36. data/lib/evilution/baseline.rb +31 -6
  37. data/lib/evilution/cache/keys.rb +140 -0
  38. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  39. data/lib/evilution/cache/skeleton.rb +57 -0
  40. data/lib/evilution/cache/spec_digest.rb +39 -0
  41. data/lib/evilution/cache.rb +143 -50
  42. data/lib/evilution/class_tree.rb +38 -0
  43. data/lib/evilution/cli/command.rb +12 -4
  44. data/lib/evilution/cli/commands/run.rb +12 -3
  45. data/lib/evilution/cli/commands/subjects.rb +5 -2
  46. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  47. data/lib/evilution/cli/printers/environment.rb +34 -4
  48. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  49. data/lib/evilution/compare/categorizer.rb +2 -1
  50. data/lib/evilution/compare/normalizer.rb +3 -2
  51. data/lib/evilution/config/sources.rb +3 -0
  52. data/lib/evilution/config/validators/base.rb +9 -0
  53. data/lib/evilution/config/validators/since.rb +22 -0
  54. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  55. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  56. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  58. data/lib/evilution/config.rb +122 -6
  59. data/lib/evilution/coverage/load_tracker.rb +50 -0
  60. data/lib/evilution/coverage/map.rb +42 -5
  61. data/lib/evilution/coverage/map_builder.rb +123 -22
  62. data/lib/evilution/coverage/map_store.rb +15 -5
  63. data/lib/evilution/coverage/recorder.rb +42 -3
  64. data/lib/evilution/coverage_example_filter.rb +35 -0
  65. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  66. data/lib/evilution/example_filter.rb +20 -0
  67. data/lib/evilution/git/changed_lines.rb +103 -0
  68. data/lib/evilution/git/since_scope.rb +77 -0
  69. data/lib/evilution/git/unified_diff.rb +68 -0
  70. data/lib/evilution/infra_crash.rb +26 -0
  71. data/lib/evilution/integration/base.rb +97 -6
  72. data/lib/evilution/integration/crash_detector.rb +46 -6
  73. data/lib/evilution/integration/known_failures.rb +4 -0
  74. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  75. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  76. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  77. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  78. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  79. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  80. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  81. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  82. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  83. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  84. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  85. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  86. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  87. data/lib/evilution/integration/minitest.rb +22 -2
  88. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  89. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  90. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  91. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  92. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  93. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  94. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  95. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  96. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  97. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  98. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  99. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  100. data/lib/evilution/integration/rspec.rb +98 -47
  101. data/lib/evilution/integration/stop_policy.rb +49 -0
  102. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  103. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  104. data/lib/evilution/integration/test_unit.rb +28 -2
  105. data/lib/evilution/isolation/fork.rb +58 -13
  106. data/lib/evilution/isolation/in_process.rb +12 -1
  107. data/lib/evilution/line_set.rb +72 -0
  108. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  109. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  110. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  111. data/lib/evilution/mcp/info_tool.rb +24 -17
  112. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  113. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  114. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  115. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  116. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  117. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  118. data/lib/evilution/mcp/session_tool.rb +1 -2
  119. data/lib/evilution/mcp.rb +4 -0
  120. data/lib/evilution/mutation/line_diff.rb +111 -0
  121. data/lib/evilution/mutation.rb +17 -13
  122. data/lib/evilution/mutator/base.rb +121 -98
  123. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  124. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  125. data/lib/evilution/mutator/load_sources.rb +88 -0
  126. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  127. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  128. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  129. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  130. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  131. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  132. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  133. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  135. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  136. data/lib/evilution/mutator/registry.rb +11 -5
  137. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  138. data/lib/evilution/parallel/pool.rb +11 -4
  139. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  140. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  141. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  142. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  143. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  144. data/lib/evilution/parallel/work_queue.rb +69 -12
  145. data/lib/evilution/parallel_db_warning.rb +39 -12
  146. data/lib/evilution/process_supervisor.rb +22 -2
  147. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  148. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  149. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  150. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  151. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  152. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  153. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  154. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  155. data/lib/evilution/reporter/cli.rb +9 -2
  156. data/lib/evilution/reporter/html/assets/style.css +8 -0
  157. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  158. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  159. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  160. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  161. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  162. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  163. data/lib/evilution/reporter/json/subjects.rb +1 -0
  164. data/lib/evilution/reporter/json/timings.rb +42 -0
  165. data/lib/evilution/reporter/json.rb +13 -50
  166. data/lib/evilution/result/coverage_gap.rb +5 -0
  167. data/lib/evilution/result/mutation_result.rb +55 -3
  168. data/lib/evilution/result/mutation_timings.rb +54 -0
  169. data/lib/evilution/result/run_timings.rb +20 -0
  170. data/lib/evilution/result/subject_score.rb +8 -1
  171. data/lib/evilution/result/subject_scorer.rb +5 -3
  172. data/lib/evilution/result/summary.rb +49 -12
  173. data/lib/evilution/runner/baseline_runner.rb +47 -5
  174. data/lib/evilution/runner/diagnostics.rb +0 -13
  175. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  176. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  177. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  178. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  179. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  180. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  181. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  182. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  183. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  184. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  185. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  186. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  187. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  190. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  191. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  195. data/lib/evilution/runner/mutation_executor.rb +75 -10
  196. data/lib/evilution/runner/mutation_planner.rb +29 -21
  197. data/lib/evilution/runner/phase_clock.rb +22 -0
  198. data/lib/evilution/runner/report_publisher.rb +1 -1
  199. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  200. data/lib/evilution/runner.rb +87 -26
  201. data/lib/evilution/session/diff.rb +9 -2
  202. data/lib/evilution/session/store.rb +17 -2
  203. data/lib/evilution/source_ast_cache.rb +10 -0
  204. data/lib/evilution/spec_ast_cache.rb +15 -0
  205. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  206. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  207. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  208. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  209. data/lib/evilution/spec_resolver.rb +76 -65
  210. data/lib/evilution/spec_selector.rb +17 -13
  211. data/lib/evilution/subject.rb +7 -2
  212. data/lib/evilution/version.rb +1 -1
  213. data/lib/evilution/worker_database/copies.rb +73 -0
  214. data/lib/evilution/worker_database/schema.rb +39 -0
  215. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  216. data/lib/evilution/worker_database.rb +214 -0
  217. data/schema/evilution.config.schema.json +74 -2
  218. metadata +84 -13
  219. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  220. data/scripts/benchmark_density +0 -262
  221. data/scripts/benchmark_density.yml +0 -19
  222. data/scripts/canary_manifest.yml +0 -47
  223. data/scripts/compare_mutations +0 -421
  224. data/scripts/compare_mutations.yml +0 -24
  225. data/scripts/compare_targeting +0 -279
  226. data/scripts/compare_targeting.example.yml +0 -24
  227. data/scripts/mutant_json_adapter +0 -227
@@ -4,92 +4,185 @@ require "digest"
4
4
  require "json"
5
5
  require "fileutils"
6
6
 
7
+ # The verdicts --incremental keeps across runs, one JSON file per source file:
8
+ #
9
+ # { "format": 2, "path": "lib/shop.rb",
10
+ # "subjects": { <subject key>: { "name": "Shop#price",
11
+ # "mutations": { <mutation key>: { "status": ..., ... } } } } }
12
+ #
13
+ # What the keys stand for is up to Cache::Keys. A file is read once per run,
14
+ # when its first mutation is looked up, and verdicts are stored in memory
15
+ # and written a file at a time: when the run moves on to the next source
16
+ # file, and at #flush. Storing a verdict therefore costs the same however
17
+ # many the file already holds.
18
+ #
19
+ # Writing a file drops what it holds for an earlier version of a subject the
20
+ # run just looked up, and for a subject the file no longer has, so a file
21
+ # holds roughly one version of each subject.
7
22
  class Evilution::Cache
8
23
  DEFAULT_DIR = "tmp/evilution_cache"
24
+ FORMAT = 2
25
+ FORMAT_DIR = "v#{FORMAT}".freeze
9
26
 
10
- def initialize(cache_dir: DEFAULT_DIR)
27
+ autoload :Keys, File.expand_path("cache/keys", __dir__)
28
+ autoload :RunFingerprint, File.expand_path("cache/run_fingerprint", __dir__)
29
+ autoload :Skeleton, File.expand_path("cache/skeleton", __dir__)
30
+ autoload :SpecDigest, File.expand_path("cache/spec_digest", __dir__)
31
+
32
+ def initialize(keys:, cache_dir: DEFAULT_DIR)
33
+ @keys = keys
11
34
  @cache_dir = cache_dir
35
+ @owner = Process.pid
36
+ reset
12
37
  end
13
38
 
14
- def fetch(mutation)
15
- return nil if stripped?(mutation)
39
+ # Every subject parsed from the run's files (see Cache::Keys#index).
40
+ def index(subjects)
41
+ @keys.index(subjects)
42
+ end
16
43
 
17
- file_key = file_key(mutation)
18
- entry_key = entry_key(mutation)
19
- data = read_file(file_key)
20
- return nil unless data
44
+ def fetch(mutation)
45
+ keys = keys_for(mutation)
46
+ return nil unless keys
21
47
 
22
- entry = data[entry_key]
23
- return nil unless entry.is_a?(Hash) && entry["status"].is_a?(String)
48
+ subject_key, mutation_key = keys
49
+ file_path = mutation.file_path
50
+ touch(file_path, subject_key, mutation.subject.name)
51
+ entry = subjects_of(file_path)[subject_key]
52
+ return nil unless entry
24
53
 
25
- { status: entry["status"].to_sym, duration: entry["duration"],
26
- killing_test: entry["killing_test"], test_command: entry["test_command"] }
54
+ from_entry(entry["mutations"][mutation_key])
27
55
  end
28
56
 
29
57
  def store(mutation, result_data)
30
- return if stripped?(mutation)
31
-
32
- file_key = file_key(mutation)
33
- entry_key = entry_key(mutation)
34
- data = read_file(file_key) || {}
58
+ keys = keys_for(mutation)
59
+ return unless keys
60
+
61
+ subject_key, mutation_key = keys
62
+ file_path = mutation.file_path
63
+ move_to(file_path)
64
+ name = mutation.subject.name
65
+ touch(file_path, subject_key, name)
66
+ entry = subjects_of(file_path)[subject_key] ||= { "name" => name, "mutations" => {} }
67
+ entry["mutations"][mutation_key] = to_entry(result_data)
68
+ @dirty[file_path] = true
69
+ end
35
70
 
36
- data[entry_key] = {
37
- "status" => result_data[:status].to_s,
38
- "duration" => result_data[:duration],
39
- "killing_test" => result_data[:killing_test],
40
- "test_command" => result_data[:test_command]
41
- }
71
+ # Writes every file stored to since it was last written. Only the process
72
+ # that made the cache writes: a forked child holds a copy of its verdicts.
73
+ def flush
74
+ return unless Process.pid == @owner
42
75
 
43
- write_file(file_key, data)
76
+ @dirty.each_key { |file_path| write(file_path) }
77
+ @dirty.clear
78
+ remove_earlier_format
44
79
  end
45
80
 
46
81
  def clear
47
82
  FileUtils.rm_rf(@cache_dir)
83
+ reset
48
84
  end
49
85
 
50
86
  private
51
87
 
52
- # Both keys digest a source, so a mutation missing either one cannot be keyed.
53
- def stripped?(mutation)
54
- mutation.original_source.nil? || mutation.mutated_source.nil?
88
+ def reset
89
+ @files = {}
90
+ @dirty = {}
91
+ @touched = Hash.new { |hash, file_path| hash[file_path] = {} }
92
+ @current = nil
55
93
  end
56
94
 
57
- def file_key(mutation)
58
- content_hash = Digest::SHA256.hexdigest(mutation.original_source)
59
- "#{safe_filename(mutation.file_path)}_#{content_hash[0, 16]}"
95
+ # [subject key, mutation key], or nil when either is missing.
96
+ def keys_for(mutation)
97
+ subject_key = @keys.subject_key(mutation)
98
+ mutation_key = subject_key && @keys.mutation_key(mutation)
99
+ mutation_key && [subject_key, mutation_key]
60
100
  end
61
101
 
62
- # Several operators emit more than one mutation at the same location (e.g.
63
- # nil_replacement -> true / false / 0), so the location alone would let a
64
- # killed sibling answer for a survivor. The original source is fixed per
65
- # cache file, so a digest of the mutated source identifies the mutation.
66
- def entry_key(mutation)
67
- mutated_digest = Digest::SHA256.hexdigest(mutation.mutated_source)
68
- "#{mutation.operator_name}:#{mutation.line}:#{mutation.column}:#{mutated_digest[0, 16]}"
102
+ def touch(file_path, subject_key, name)
103
+ @touched[file_path][subject_key] = name
69
104
  end
70
105
 
71
- def safe_filename(path)
72
- path.gsub(%r{[/\\]}, "_").gsub(/[^a-zA-Z0-9._-]/, "")
106
+ # Mutations come file by file, so a file is done once the next one starts;
107
+ # writing it then keeps what a run that is cut short had already judged.
108
+ def move_to(file_path)
109
+ previous = @current
110
+ @current = file_path
111
+ return if previous == file_path || !@dirty.delete(previous)
112
+
113
+ write(previous) if Process.pid == @owner
114
+ end
115
+
116
+ def to_entry(data)
117
+ {
118
+ "status" => data[:status].to_s,
119
+ "duration" => data[:duration],
120
+ "killing_test" => data[:killing_test],
121
+ "test_command" => data[:test_command],
122
+ "time_limit" => data[:time_limit]
123
+ }
73
124
  end
74
125
 
75
- def read_file(file_key)
76
- path = cache_path(file_key)
77
- return nil unless File.exist?(path)
126
+ # The reports round the duration and time limit, so an entry whose
127
+ # duration is not a number is no entry, and such a time limit is none.
128
+ def from_entry(entry)
129
+ return nil unless entry.is_a?(Hash) && entry["status"].is_a?(String) && entry["duration"].is_a?(Numeric)
78
130
 
79
- JSON.parse(File.read(path))
80
- rescue JSON::ParserError
81
- nil
131
+ time_limit = entry["time_limit"]
132
+ { status: entry["status"].to_sym, duration: entry["duration"], killing_test: entry["killing_test"],
133
+ test_command: entry["test_command"], time_limit: time_limit.is_a?(Numeric) ? time_limit : nil }
82
134
  end
83
135
 
84
- def write_file(file_key, data)
85
- FileUtils.mkdir_p(@cache_dir)
86
- path = cache_path(file_key)
136
+ def subjects_of(file_path)
137
+ @files[file_path] ||= read(file_path)
138
+ end
139
+
140
+ # What a file holds, with anything not of the expected shape left out. A
141
+ # source file with no cache file yet holds nothing.
142
+ def read(file_path)
143
+ data = JSON.parse(File.read(cache_path(file_path)))
144
+ return {} unless data.is_a?(Hash) && data["format"] == FORMAT && data["subjects"].is_a?(Hash)
145
+
146
+ data["subjects"].select { |_key, entry| entry.is_a?(Hash) && entry["mutations"].is_a?(Hash) }
147
+ rescue JSON::ParserError, SystemCallError
148
+ {}
149
+ end
150
+
151
+ def write(file_path)
152
+ subjects = current_subjects(file_path)
153
+ path = cache_path(file_path)
154
+ FileUtils.mkdir_p(File.dirname(path))
87
155
  tmp = "#{path}.#{Process.pid}.tmp"
88
- File.write(tmp, JSON.generate(data))
156
+ File.write(tmp, JSON.generate("format" => FORMAT, "path" => file_path, "subjects" => subjects))
89
157
  File.rename(tmp, path)
90
158
  end
91
159
 
92
- def cache_path(file_key)
93
- File.join(@cache_dir, "#{file_key}.json")
160
+ # A subject's entry stays when the run looked it up, or when the run
161
+ # neither looked up another version of it nor found the file without it.
162
+ def current_subjects(file_path)
163
+ touched = @touched[file_path]
164
+ touched_names = touched.values
165
+ known = @keys.subject_names(file_path)
166
+ subjects_of(file_path).select do |subject_key, entry|
167
+ next true if touched.key?(subject_key)
168
+
169
+ name = entry["name"]
170
+ !touched_names.include?(name) && (known.nil? || known.include?(name))
171
+ end
172
+ end
173
+
174
+ # The cache files of format 1 sat directly in the cache directory, one per
175
+ # version of a source file, and are never read again.
176
+ def remove_earlier_format
177
+ FileUtils.rm_f(Dir.glob(File.join(@cache_dir, "*.json")))
178
+ end
179
+
180
+ def cache_path(file_path)
181
+ name = "#{safe_filename(file_path)}-#{Digest::SHA256.hexdigest(file_path)[0, 12]}.json"
182
+ File.join(@cache_dir, FORMAT_DIR, name)
183
+ end
184
+
185
+ def safe_filename(path)
186
+ path.gsub(%r{[/\\]}, "_").gsub(/[^a-zA-Z0-9._-]/, "")
94
187
  end
95
188
  end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "version"
4
+
5
+ # The classes below a class, reached through each class's own list of
6
+ # subclasses.
7
+ #
8
+ # ObjectSpace.each_object(Class) finds them too, but by walking the whole
9
+ # heap: every object of the process is visited to pick out the classes. In a
10
+ # forked mutation process that is every page the child shares with its
11
+ # parent, which the walk forces it to map; under Rails, millions of objects
12
+ # for every mutation. The class tree holds only the classes.
13
+ #
14
+ # Singleton classes are not in it, which is what every caller wants.
15
+ #
16
+ # Class#subclasses is called through an UnboundMethod: a class may define a
17
+ # `subclasses` of its own, which need not answer the question asked here.
18
+ module Evilution::ClassTree
19
+ SUBCLASSES = Class.instance_method(:subclasses)
20
+ private_constant :SUBCLASSES
21
+
22
+ # Every class below `root`, at any depth; `root` itself is left out.
23
+ def self.descendants(root)
24
+ found = []
25
+ visit(root) { |klass| found << klass }
26
+ found.shift
27
+ found
28
+ end
29
+
30
+ # Yields `root`, then the classes below it, each before its own
31
+ # subclasses. Where the block answers falsy, the classes below the one it
32
+ # was given are passed over.
33
+ def self.visit(root, &block)
34
+ return unless yield(root)
35
+
36
+ SUBCLASSES.bind_call(root).each { |klass| visit(klass, &block) }
37
+ end
38
+ end
@@ -2,6 +2,8 @@
2
2
 
3
3
  require_relative "../cli"
4
4
  require_relative "result"
5
+ require_relative "../config"
6
+ require_relative "../git/since_scope"
5
7
 
6
8
  class Evilution::CLI::Command
7
9
  def initialize(parsed_args, stdout: $stdout, stderr: $stderr)
@@ -25,14 +27,20 @@ class Evilution::CLI::Command
25
27
  raise NotImplementedError
26
28
  end
27
29
 
30
+ # The file arguments and their line ranges, limited with --since REF to the
31
+ # lines changed since that ref. The ref is checked before git reads it;
32
+ # Config, given the same option, strips it the same way.
33
+ def scoped_paths
34
+ since = Evilution::Config::Validators::Since.call(@options[:since], target: @options[:target])
35
+ Evilution::Git::SinceScope.apply(since, files: @files, line_ranges: @line_ranges)
36
+ end
37
+
28
38
  def build_operator_options(config)
29
39
  { skip_heredoc_literals: config.skip_heredoc_literals? }
30
40
  end
31
41
 
32
42
  def build_subject_filter(config)
33
- return nil if config.ignore_patterns.empty?
34
-
35
- require_relative "../ast/pattern/filter"
36
- Evilution::AST::Pattern::Filter.new(config.ignore_patterns)
43
+ require_relative "../ast/skip_filter"
44
+ Evilution::AST::SkipFilter.for(config)
37
45
  end
38
46
  end
@@ -18,18 +18,27 @@ class Evilution::CLI::Commands::Run < Evilution::CLI::Command
18
18
  config = nil
19
19
  raise Evilution::ConfigError, @stdin_error if @stdin_error
20
20
 
21
- config = Evilution::Config.new(**@options, target_files: @files, line_ranges: @line_ranges)
21
+ paths = scoped_paths
22
+ config = Evilution::Config.new(**@options, target_files: paths.files, line_ranges: paths.ranges)
22
23
  hooks = build_hooks(config)
23
24
  runner = Evilution::Runner.new(config: config, hooks: hooks)
24
25
  summary = runner.call
25
- exit_code = summary.success?(min_score: config.min_score) ? 0 : 1
26
- Evilution::CLI::Result.new(exit_code: exit_code)
26
+ Evilution::CLI::Result.new(exit_code: passed?(summary, config) ? 0 : 1)
27
27
  rescue Evilution::Error => e
28
28
  handle_error(e, config, file_options)
29
29
  end
30
30
 
31
31
  private
32
32
 
33
+ # A --since run whose changed lines hold nothing to mutate has nothing to
34
+ # score: a pull request that touches no Ruby code passes. A target file that
35
+ # resolved to no spec still fails it, as the result line says.
36
+ def passed?(summary, config)
37
+ return true if config.since? && summary.total.zero? && !summary.unresolved_targets?
38
+
39
+ summary.success?(min_score: config.min_score)
40
+ end
41
+
33
42
  def handle_error(error, config, file_options)
34
43
  if json_format?(config, file_options)
35
44
  @stdout.puts(JSON.generate(error_payload(error)))
@@ -17,7 +17,8 @@ class Evilution::CLI::Commands::Subjects < Evilution::CLI::Command
17
17
  def perform
18
18
  raise Evilution::ConfigError, @stdin_error if @stdin_error
19
19
 
20
- config = Evilution::Config.new(target_files: @files, line_ranges: @line_ranges, **@options)
20
+ paths = scoped_paths
21
+ config = Evilution::Config.new(**@options, target_files: paths.files, line_ranges: paths.ranges)
21
22
  runner = Evilution::Runner.new(config: config)
22
23
  subjects = runner.parse_and_filter_subjects
23
24
 
@@ -34,7 +35,9 @@ class Evilution::CLI::Commands::Subjects < Evilution::CLI::Command
34
35
  def collect_entries(subjects, config)
35
36
  registry = Evilution::Mutator::Registry.default
36
37
  filter = build_subject_filter(config)
37
- operator_options = build_operator_options(config)
38
+ # Only mutations that change a line of the requested range are counted,
39
+ # as only those would run.
40
+ operator_options = build_operator_options(config).merge(line_ranges: config.line_ranges)
38
41
  entries = []
39
42
  total = 0
40
43
 
@@ -45,13 +45,30 @@ class Evilution::CLI::Parser::OptionsBuilder
45
45
 
46
46
  def add_core_options(opts)
47
47
  opts.on("-j", "--jobs N", Integer, "Number of parallel workers (default: 1)") { |n| @options[:jobs] = n }
48
- opts.on("-t", "--timeout N", Integer, "Per-mutation timeout in seconds") { |n| @options[:timeout] = n }
48
+ opts.on("-t", "--timeout N", Integer,
49
+ "Per-mutation timeout in seconds, the same for every mutation",
50
+ "(default: derived from the baseline, see --timeout-factor)") { |n| @options[:timeout] = n }
51
+ add_adaptive_timeout_options(opts)
49
52
  opts.on("-f", "--format FORMAT", "Output format: text, json, html") { |f| @options[:format] = f.to_sym }
50
53
  end
51
54
 
55
+ def add_adaptive_timeout_options(opts)
56
+ opts.on("--timeout-factor K", Float,
57
+ "Without --timeout: limit each mutation to K x the baseline time of its spec files (default: 10)") do |k|
58
+ @options[:timeout_factor] = k
59
+ end
60
+ opts.on("--timeout-floor N", Float, "Without --timeout: shortest such limit in seconds (default: 10)") do |n|
61
+ @options[:timeout_floor] = n
62
+ end
63
+ opts.on("--timeout-ceiling N", Float, "Without --timeout: longest such limit in seconds (default: 60)") do |n|
64
+ @options[:timeout_ceiling] = n
65
+ end
66
+ end
67
+
52
68
  def add_filter_options(opts)
53
69
  add_spec_filter_options(opts)
54
70
  add_targeting_options(opts)
71
+ add_uncovered_options(opts)
55
72
  end
56
73
 
57
74
  def add_spec_filter_options(opts)
@@ -84,6 +101,19 @@ class Evilution::CLI::Parser::OptionsBuilder
84
101
  end
85
102
  end
86
103
 
104
+ def add_uncovered_options(opts)
105
+ opts.on("--[no-]skip-uncovered",
106
+ "With --example-targeting coverage, report a mutation as uncovered without running it",
107
+ "when no example executes any line it changes (default: enabled)") do |v|
108
+ @options[:skip_uncovered] = v
109
+ end
110
+ opts.on("--[no-]probe-first",
111
+ "Experimental: run each method's body-to-raise mutant first; when it survives, report the",
112
+ "method's other mutants as survivors implied by it, without running them (default: disabled)") do |v|
113
+ @options[:probe_first] = v
114
+ end
115
+ end
116
+
87
117
  # Map the single --example-targeting flag onto the two underlying config axes:
88
118
  # full_file is "targeting off" (run all resolved examples); lexical/coverage
89
119
  # turn targeting on and select the strategy.
@@ -101,7 +131,7 @@ class Evilution::CLI::Parser::OptionsBuilder
101
131
  "(default: disabled; if provided without N, uses 1; use --fail-fast=N)") { @options[:fail_fast] ||= 1 }
102
132
  opts.on("--no-baseline", "Skip baseline test suite check") { @options[:baseline] = false }
103
133
  opts.on("--[no-]incremental",
104
- "Cache killed/timeout results; skip re-running them on unchanged files. " \
134
+ "Cache verdicts per method; skip re-running them until the method or its specs change. " \
105
135
  "Use --no-incremental to override `incremental: true` from the config file for one run.") do |v|
106
136
  @options[:incremental] = v
107
137
  end
@@ -110,6 +140,12 @@ class Evilution::CLI::Parser::OptionsBuilder
110
140
  "the pipeline misreports it (default: enabled). Use --no-canary to skip.") do |v|
111
141
  @options[:canary] = v
112
142
  end
143
+ opts.on("--[no-]stop-on-kill",
144
+ "Stop each mutation's test run at the first failure that decides the verdict " \
145
+ "(default: enabled). Use --no-stop-on-kill to run every selected example.") do |v|
146
+ @options[:stop_on_kill] = v
147
+ end
148
+ add_killer_first_option(opts)
113
149
  opts.on("--stdin", "Read target file paths from stdin (one per line)") { @options[:stdin] = true }
114
150
  end
115
151
 
@@ -122,6 +158,15 @@ class Evilution::CLI::Parser::OptionsBuilder
122
158
  opts.on("--no-preload", "Disable parent-process preload even for Rails projects") { @options[:preload] = false }
123
159
  opts.on("--warmup NAME", "After preload, warm lazily-initialised framework state once " \
124
160
  "in the parent: none, rails (default: none)") { |w| @options[:warmup] = w }
161
+ add_worker_database_options(opts)
162
+ end
163
+
164
+ def add_worker_database_options(opts)
165
+ opts.on("--[no-]copy-worker-databases",
166
+ "With --jobs > 1 and a preloaded Rails app, give each worker its own copy of a SQLite",
167
+ "test database the workers would otherwise share (default: enabled)") do |v|
168
+ @options[:copy_worker_databases] = v
169
+ end
125
170
  end
126
171
 
127
172
  def add_output_options(opts)
@@ -144,6 +189,14 @@ class Evilution::CLI::Parser::OptionsBuilder
144
189
  opts.on("--strict", "Shortcut for --profile=strict") { @options[:profile] = "strict" }
145
190
  end
146
191
 
192
+ def add_killer_first_option(opts)
193
+ opts.on("--[no-]killer-first",
194
+ "With RSpec and stop-on-kill, run first the examples that killed nearby mutations, then the " \
195
+ "cheapest (default: disabled). Only for suites whose examples pass in any order.") do |v|
196
+ @options[:killer_first] = v
197
+ end
198
+ end
199
+
147
200
  def add_extra_flag_options(opts)
148
201
  add_mutation_behavior_options(opts)
149
202
  add_session_persistence_options(opts)
@@ -152,6 +205,11 @@ class Evilution::CLI::Parser::OptionsBuilder
152
205
 
153
206
  def add_mutation_behavior_options(opts)
154
207
  opts.on("--skip-heredoc-literals", "Skip all string literal mutations inside heredocs") { @options[:skip_heredoc_literals] = true }
208
+ opts.on("--[no-]skip-sites PRESETS", Array,
209
+ "Leave low-value code unmutated, comma-separated: logging, debug_output, error_messages,",
210
+ "requires (default: none). --no-skip-sites overrides skip_sites from the config file") do |presets|
211
+ @options[:skip_sites] = presets || []
212
+ end
155
213
  opts.on("--related-specs-heuristic", "Append related request/integration/feature/system specs for includes() mutations") do
156
214
  @options[:related_specs_heuristic] = true
157
215
  end
@@ -176,7 +234,13 @@ class Evilution::CLI::Parser::OptionsBuilder
176
234
  def add_session_options(opts)
177
235
  opts.on("--results-dir DIR", "Session results directory") { |d| @options[:results_dir] = d }
178
236
  opts.on("--limit N", Integer, "Show only the N most recent sessions") { |n| @options[:limit] = n }
179
- opts.on("--since DATE", "Show sessions since DATE (YYYY-MM-DD)") { |d| @options[:since] = d }
237
+ # One flag read per command: a git ref where files are mutated, a date
238
+ # where sessions are listed.
239
+ opts.on("--since REF|DATE", "run, subjects: mutate only the lines changed since git REF",
240
+ "(from its merge base with HEAD, uncommitted changes included);",
241
+ "session list: sessions since DATE (YYYY-MM-DD)") do |value|
242
+ @options[:since] = value
243
+ end
180
244
  opts.on("--older-than DURATION", "Delete sessions older than DURATION (e.g., 30d, 24h, 1w)") do |d|
181
245
  @options[:older_than] = d
182
246
  end
@@ -8,6 +8,7 @@ class Evilution::CLI::Printers::Environment
8
8
  verbose quiet progress min_score suggest_tests save_session
9
9
  skip_heredoc_literals
10
10
  ].freeze
11
+ RUN_SETTINGS = %i[killer_first skip_uncovered probe_first copy_worker_databases].freeze
11
12
 
12
13
  def initialize(config, config_file:)
13
14
  @config = config
@@ -36,11 +37,36 @@ class Evilution::CLI::Printers::Environment
36
37
  end
37
38
 
38
39
  def settings_lines
39
- plain_lines = PLAIN_SETTINGS.map { |k| setting_line(k, @config.public_send(k)) }
40
+ plain_lines = plain_setting_lines
40
41
  plain_lines.insert(10, setting_line(:fail_fast, @config.fail_fast || "(disabled)"))
41
- plain_lines.insert(14, setting_line(:target, @config.target || "(all files)"))
42
- plain_lines << setting_line(:ignore_patterns, format_ignore_patterns(@config.ignore_patterns))
43
- plain_lines
42
+ plain_lines.insert(11, setting_line(:stop_on_kill, @config.stop_on_kill))
43
+ plain_lines.insert(15, setting_line(:target, @config.target || "(all files)"))
44
+ plain_lines.concat(run_lines)
45
+ plain_lines.concat(skip_lines)
46
+ end
47
+
48
+ def plain_setting_lines
49
+ PLAIN_SETTINGS.map { |k| setting_line(k, k == :timeout ? timeout_value : @config.public_send(k)) }
50
+ end
51
+
52
+ def run_lines
53
+ RUN_SETTINGS.map { |k| setting_line(k, @config.public_send(k)) }
54
+ end
55
+
56
+ # With no timeout set, the configured one is only the fallback: each
57
+ # mutation's limit follows the baseline time of its spec files.
58
+ def timeout_value
59
+ return @config.timeout unless @config.adaptive_timeout?
60
+
61
+ "adaptive (#{@config.timeout_factor} x baseline, #{@config.timeout_floor}-#{@config.timeout_ceiling} s; " \
62
+ "#{@config.timeout} s without a baseline time)"
63
+ end
64
+
65
+ def skip_lines
66
+ [
67
+ setting_line(:ignore_patterns, format_ignore_patterns(@config.ignore_patterns)),
68
+ setting_line(:skip_sites, format_skip_sites(@config.skip_sites))
69
+ ]
44
70
  end
45
71
 
46
72
  def setting_line(key, value)
@@ -50,4 +76,8 @@ class Evilution::CLI::Printers::Environment
50
76
  def format_ignore_patterns(patterns)
51
77
  patterns.empty? ? "(none)" : patterns.inspect
52
78
  end
79
+
80
+ def format_skip_sites(presets)
81
+ presets.empty? ? "(none)" : presets.join(", ")
82
+ end
53
83
  end
@@ -23,6 +23,7 @@ class Evilution::CLI::Printers::SessionDetail
23
23
  print_header(io, @data)
24
24
  print_summary(io, @data["summary"])
25
25
  print_survived(io, @data["survived"] || [])
26
+ print_uncovered(io, @data["uncovered"] || [])
26
27
  end
27
28
 
28
29
  def print_header(io, data)
@@ -45,15 +46,22 @@ class Evilution::CLI::Printers::SessionDetail
45
46
  io.puts("")
46
47
  io.puts(
47
48
  format(
48
- "Score: %<score>.2f%% Total: %<total>d Killed: %<killed>d Survived: %<surv>d " \
49
+ "Score: %<score>.2f%% Total: %<total>d Killed: %<killed>d Survived: %<surv>d %<uncovered>s" \
49
50
  "Timed out: %<to>d Errors: %<err>d Duration: %<dur>.1fs",
50
51
  score: summary["score"] * 100, total: summary["total"], killed: summary["killed"],
51
- surv: summary["survived"], to: summary["timed_out"], err: summary["errors"],
52
- dur: summary["duration"]
52
+ surv: summary["survived"], uncovered: uncovered_count(summary), to: summary["timed_out"],
53
+ err: summary["errors"], dur: summary["duration"]
53
54
  )
54
55
  )
55
56
  end
56
57
 
58
+ # Shown only when there are any, so a session from before they were
59
+ # recorded reads as it did.
60
+ def uncovered_count(summary)
61
+ count = summary["uncovered"].to_i
62
+ count.positive? ? "Uncovered: #{count} " : ""
63
+ end
64
+
57
65
  def print_survived(io, survived)
58
66
  io.puts("")
59
67
  if survived.empty?
@@ -64,6 +72,15 @@ class Evilution::CLI::Printers::SessionDetail
64
72
  end
65
73
  end
66
74
 
75
+ # The mutations no test executes: survivors that were not run.
76
+ def print_uncovered(io, uncovered)
77
+ return if uncovered.empty?
78
+
79
+ io.puts("")
80
+ io.puts("Uncovered mutations (#{uncovered.length}, no test executes the lines they change):")
81
+ uncovered.each_with_index { |m, i| print_mutation_detail(io, m, i + 1) }
82
+ end
83
+
67
84
  def print_mutation_detail(io, mutation, index)
68
85
  io.puts("")
69
86
  io.puts(" #{index}. #{mutation["operator"]} — #{mutation["file"]}:#{mutation["line"]}")
@@ -4,7 +4,8 @@ require_relative "../compare"
4
4
  require_relative "record"
5
5
 
6
6
  module Evilution::Compare::Categorizer
7
- ALIVE = %i[survived].freeze
7
+ # An uncovered mutation is a survivor that was not run: no test executes it.
8
+ ALIVE = %i[survived uncovered].freeze
8
9
  DEAD = %i[killed timeout error].freeze
9
10
  # neutral, equivalent, unresolved, unparseable are non-actionable signals
10
11
  # — excluded from alive/dead buckets, counted in summary.
@@ -8,7 +8,7 @@ require_relative "diff_extractor/evilution"
8
8
  require_relative "diff_extractor/mutant"
9
9
 
10
10
  class Evilution::Compare::Normalizer
11
- EVILUTION_BUCKETS = %w[killed survived timed_out errors neutral equivalent unresolved unparseable].freeze
11
+ EVILUTION_BUCKETS = %w[killed survived timed_out errors neutral equivalent unresolved unparseable uncovered].freeze
12
12
  EVILUTION_STATUS_MAP = {
13
13
  "killed" => :killed,
14
14
  "survived" => :survived,
@@ -17,7 +17,8 @@ class Evilution::Compare::Normalizer
17
17
  "neutral" => :neutral,
18
18
  "equivalent" => :equivalent,
19
19
  "unresolved" => :unresolved,
20
- "unparseable" => :unparseable
20
+ "unparseable" => :unparseable,
21
+ "uncovered" => :uncovered
21
22
  }.freeze
22
23
 
23
24
  def initialize
@@ -7,9 +7,12 @@ require_relative "env_loader"
7
7
  module Evilution::Config::Sources
8
8
  module_function
9
9
 
10
+ # timeout_given: whether the file or the caller set a timeout, which then
11
+ # applies to every mutation instead of one derived from the baseline.
10
12
  def merge(explicit:, skip_file:)
11
13
  file = skip_file ? {} : Evilution::Config::FileLoader.load
12
14
  env = Evilution::Config::EnvLoader.load
13
15
  Evilution::Config::DEFAULTS.merge(file).merge(env).merge(explicit)
16
+ .merge(timeout_given: file.key?(:timeout) || explicit.key?(:timeout))
14
17
  end
15
18
  end
@@ -33,5 +33,14 @@ class Evilution::Config::Validators::Base
33
33
  rescue ::ArgumentError, ::TypeError
34
34
  raise Evilution::ConfigError, "#{name} must be a positive integer, got #{value.inspect}"
35
35
  end
36
+
37
+ # A number given as a number keeps its type, so it reads back as written;
38
+ # a string is parsed as a Float.
39
+ def coerce_positive_number!(value, name:)
40
+ number = value.is_a?(Numeric) ? value : Float(value, exception: false)
41
+ return number if number && number.finite? && number.positive?
42
+
43
+ raise Evilution::ConfigError, "#{name} must be a positive number, got #{value.inspect}"
44
+ end
36
45
  end
37
46
  end