evilution 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +1 -0
  3. data/.rubocop_todo.yml +30 -0
  4. data/CHANGELOG.md +59 -0
  5. data/README.md +285 -45
  6. data/docs/architecture.md +110 -12
  7. data/docs/ast_pattern_syntax.md +35 -7
  8. data/docs/combined_runs_spike.md +407 -0
  9. data/docs/integrations.md +1 -1
  10. data/docs/isolation.md +63 -2
  11. data/docs/migration-from-mutant.md +16 -11
  12. data/docs/mutant_schemata_spike.md +387 -0
  13. data/docs/operator_yield_study.md +576 -0
  14. data/docs/probe_first_spike.md +264 -0
  15. data/lib/evilution/ast/ancestor_lookup.rb +36 -0
  16. data/lib/evilution/ast/callback_declaration.rb +36 -3
  17. data/lib/evilution/ast/class_body.rb +62 -16
  18. data/lib/evilution/ast/def_region.rb +130 -0
  19. data/lib/evilution/ast/local_declaration.rb +37 -0
  20. data/lib/evilution/ast/parent_index.rb +50 -0
  21. data/lib/evilution/ast/parse_cache.rb +27 -0
  22. data/lib/evilution/ast/parser.rb +8 -3
  23. data/lib/evilution/ast/pattern/filter.rb +53 -12
  24. data/lib/evilution/ast/skip_filter.rb +83 -0
  25. data/lib/evilution/ast/skip_sites/debug_output.rb +67 -0
  26. data/lib/evilution/ast/skip_sites/error_messages.rb +74 -0
  27. data/lib/evilution/ast/skip_sites/logging.rb +93 -0
  28. data/lib/evilution/ast/skip_sites/message.rb +44 -0
  29. data/lib/evilution/ast/skip_sites/nodes.rb +105 -0
  30. data/lib/evilution/ast/skip_sites/presets.rb +31 -0
  31. data/lib/evilution/ast/skip_sites/purity.rb +69 -0
  32. data/lib/evilution/ast/skip_sites/requires.rb +40 -0
  33. data/lib/evilution/ast/skip_sites.rb +9 -0
  34. data/lib/evilution/ast/source_surgeon.rb +20 -2
  35. data/lib/evilution/ast/syntax_check.rb +10 -8
  36. data/lib/evilution/baseline/report.rb +11 -2
  37. data/lib/evilution/baseline.rb +31 -6
  38. data/lib/evilution/cache/keys.rb +140 -0
  39. data/lib/evilution/cache/run_fingerprint.rb +51 -0
  40. data/lib/evilution/cache/skeleton.rb +57 -0
  41. data/lib/evilution/cache/spec_digest.rb +39 -0
  42. data/lib/evilution/cache.rb +145 -42
  43. data/lib/evilution/class_tree.rb +38 -0
  44. data/lib/evilution/cli/command.rb +12 -4
  45. data/lib/evilution/cli/commands/run.rb +12 -3
  46. data/lib/evilution/cli/commands/subjects.rb +5 -2
  47. data/lib/evilution/cli/parser/options_builder.rb +67 -3
  48. data/lib/evilution/cli/printers/environment.rb +34 -4
  49. data/lib/evilution/cli/printers/session_detail.rb +20 -3
  50. data/lib/evilution/compare/categorizer.rb +2 -1
  51. data/lib/evilution/compare/normalizer.rb +3 -2
  52. data/lib/evilution/config/sources.rb +3 -0
  53. data/lib/evilution/config/validators/base.rb +9 -0
  54. data/lib/evilution/config/validators/since.rb +22 -0
  55. data/lib/evilution/config/validators/skip_sites.rb +24 -0
  56. data/lib/evilution/config/validators/timeout_ceiling.rb +10 -0
  57. data/lib/evilution/config/validators/timeout_factor.rb +10 -0
  58. data/lib/evilution/config/validators/timeout_floor.rb +10 -0
  59. data/lib/evilution/config.rb +122 -6
  60. data/lib/evilution/coverage/load_tracker.rb +50 -0
  61. data/lib/evilution/coverage/map.rb +42 -5
  62. data/lib/evilution/coverage/map_builder.rb +123 -22
  63. data/lib/evilution/coverage/map_store.rb +15 -5
  64. data/lib/evilution/coverage/recorder.rb +42 -3
  65. data/lib/evilution/coverage_example_filter.rb +35 -0
  66. data/lib/evilution/equivalent/heuristic/comment_marking.rb +14 -1
  67. data/lib/evilution/example_filter.rb +20 -0
  68. data/lib/evilution/git/changed_lines.rb +103 -0
  69. data/lib/evilution/git/since_scope.rb +77 -0
  70. data/lib/evilution/git/unified_diff.rb +68 -0
  71. data/lib/evilution/infra_crash.rb +26 -0
  72. data/lib/evilution/integration/base.rb +97 -6
  73. data/lib/evilution/integration/crash_detector.rb +46 -6
  74. data/lib/evilution/integration/known_failures.rb +4 -0
  75. data/lib/evilution/integration/loading/body_call_neutralizer.rb +49 -11
  76. data/lib/evilution/integration/loading/callback_equivalence.rb +62 -0
  77. data/lib/evilution/integration/loading/callback_redeclaration.rb +151 -36
  78. data/lib/evilution/integration/loading/concern_redeclaration.rb +16 -12
  79. data/lib/evilution/integration/loading/concern_registry.rb +87 -0
  80. data/lib/evilution/integration/loading/concern_state_cleaner.rb +19 -8
  81. data/lib/evilution/integration/loading/created_modules.rb +56 -0
  82. data/lib/evilution/integration/loading/frozen_namespaces.rb +80 -0
  83. data/lib/evilution/integration/loading/mutation_applier.rb +42 -11
  84. data/lib/evilution/integration/loading/neutralization.rb +41 -0
  85. data/lib/evilution/integration/loading/redefinition_recovery.rb +5 -3
  86. data/lib/evilution/integration/loading/superseded_modules.rb +22 -19
  87. data/lib/evilution/integration/minitest/stop_hook.rb +57 -0
  88. data/lib/evilution/integration/minitest.rb +22 -2
  89. data/lib/evilution/integration/minitest_crash_detector.rb +19 -0
  90. data/lib/evilution/integration/rspec/baseline_runner.rb +3 -2
  91. data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +6 -1
  92. data/lib/evilution/integration/rspec/example_filter_applier.rb +16 -0
  93. data/lib/evilution/integration/rspec/example_ids.rb +23 -0
  94. data/lib/evilution/integration/rspec/example_order.rb +86 -0
  95. data/lib/evilution/integration/rspec/killer_first.rb +72 -0
  96. data/lib/evilution/integration/rspec/result_builder.rb +7 -0
  97. data/lib/evilution/integration/rspec/state_guard/example_group_descendants.rb +44 -0
  98. data/lib/evilution/integration/rspec/state_guard.rb +2 -2
  99. data/lib/evilution/integration/rspec/target_resolver.rb +42 -0
  100. data/lib/evilution/integration/rspec/test_file_resolver.rb +4 -2
  101. data/lib/evilution/integration/rspec.rb +98 -47
  102. data/lib/evilution/integration/stop_policy.rb +49 -0
  103. data/lib/evilution/integration/test_unit/dispatcher.rb +20 -1
  104. data/lib/evilution/integration/test_unit/subject_class_registry.rb +6 -1
  105. data/lib/evilution/integration/test_unit.rb +28 -2
  106. data/lib/evilution/isolation/fork.rb +58 -13
  107. data/lib/evilution/isolation/in_process.rb +12 -1
  108. data/lib/evilution/line_set.rb +72 -0
  109. data/lib/evilution/mcp/info_tool/actions/environment.rb +12 -16
  110. data/lib/evilution/mcp/info_tool/actions/subjects.rb +3 -5
  111. data/lib/evilution/mcp/info_tool/status_glossary.rb +15 -2
  112. data/lib/evilution/mcp/info_tool.rb +24 -17
  113. data/lib/evilution/mcp/mutate_tool/config_builder.rb +13 -1
  114. data/lib/evilution/mcp/mutate_tool/option_parser.rb +2 -2
  115. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +1 -1
  116. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +28 -5
  117. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +12 -7
  118. data/lib/evilution/mcp/mutate_tool.rb +32 -7
  119. data/lib/evilution/mcp/session_tool.rb +1 -2
  120. data/lib/evilution/mcp.rb +4 -0
  121. data/lib/evilution/mutation/line_diff.rb +111 -0
  122. data/lib/evilution/mutation.rb +17 -13
  123. data/lib/evilution/mutator/base.rb +121 -98
  124. data/lib/evilution/mutator/constant_value_subjects.rb +29 -1
  125. data/lib/evilution/mutator/line_range_filter.rb +27 -0
  126. data/lib/evilution/mutator/load_sources.rb +88 -0
  127. data/lib/evilution/mutator/operator/alias_removal.rb +2 -5
  128. data/lib/evilution/mutator/operator/argument_method_call_replacement.rb +3 -0
  129. data/lib/evilution/mutator/operator/attr_accessor_swap.rb +2 -5
  130. data/lib/evilution/mutator/operator/constant_namespace_strip.rb +3 -3
  131. data/lib/evilution/mutator/operator/freeze_removal.rb +29 -11
  132. data/lib/evilution/mutator/operator/method_call_removal.rb +3 -1
  133. data/lib/evilution/mutator/operator/mixin_removal.rb +2 -5
  134. data/lib/evilution/mutator/operator/superclass_removal.rb +2 -5
  135. data/lib/evilution/mutator/operator/symbol_literal.rb +1 -1
  136. data/lib/evilution/mutator/operator/visibility_removal.rb +2 -5
  137. data/lib/evilution/mutator/registry.rb +11 -5
  138. data/lib/evilution/mutator/restore_source_cache.rb +41 -0
  139. data/lib/evilution/parallel/pool.rb +11 -4
  140. data/lib/evilution/parallel/work_queue/dispatcher/result_stream.rb +40 -0
  141. data/lib/evilution/parallel/work_queue/dispatcher.rb +20 -7
  142. data/lib/evilution/parallel/work_queue/worker/loop.rb +7 -2
  143. data/lib/evilution/parallel/work_queue/worker.rb +10 -3
  144. data/lib/evilution/parallel/work_queue/worker_stat.rb +7 -0
  145. data/lib/evilution/parallel/work_queue.rb +69 -12
  146. data/lib/evilution/parallel_db_warning.rb +39 -12
  147. data/lib/evilution/process_supervisor.rb +22 -2
  148. data/lib/evilution/reporter/cli/item_formatters/coverage_gap.rb +13 -2
  149. data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +5 -1
  150. data/lib/evilution/reporter/cli/line_formatters/mutations.rb +9 -2
  151. data/lib/evilution/reporter/cli/line_formatters/result_line.rb +9 -1
  152. data/lib/evilution/reporter/cli/line_formatters/skipped_sites.rb +14 -0
  153. data/lib/evilution/reporter/cli/line_formatters/uncovered_code.rb +17 -0
  154. data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
  155. data/lib/evilution/reporter/cli/trailer.rb +4 -4
  156. data/lib/evilution/reporter/cli.rb +9 -2
  157. data/lib/evilution/reporter/html/assets/style.css +8 -0
  158. data/lib/evilution/reporter/html/sections/file_section.rb +9 -0
  159. data/lib/evilution/reporter/html/sections/uncovered_details.rb +27 -0
  160. data/lib/evilution/reporter/html/templates/file_section.html.erb +2 -1
  161. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  162. data/lib/evilution/reporter/html/templates/uncovered_details.html.erb +11 -0
  163. data/lib/evilution/reporter/json/mutation_detail.rb +77 -0
  164. data/lib/evilution/reporter/json/subjects.rb +1 -0
  165. data/lib/evilution/reporter/json/timings.rb +42 -0
  166. data/lib/evilution/reporter/json.rb +13 -50
  167. data/lib/evilution/result/coverage_gap.rb +5 -0
  168. data/lib/evilution/result/mutation_result.rb +55 -3
  169. data/lib/evilution/result/mutation_timings.rb +54 -0
  170. data/lib/evilution/result/run_timings.rb +20 -0
  171. data/lib/evilution/result/subject_score.rb +8 -1
  172. data/lib/evilution/result/subject_scorer.rb +5 -3
  173. data/lib/evilution/result/summary.rb +49 -12
  174. data/lib/evilution/runner/baseline_runner.rb +47 -5
  175. data/lib/evilution/runner/diagnostics.rb +0 -13
  176. data/lib/evilution/runner/isolation_resolver.rb +16 -4
  177. data/lib/evilution/runner/mutation_executor/isolated_run.rb +34 -0
  178. data/lib/evilution/runner/mutation_executor/kill_history.rb +66 -0
  179. data/lib/evilution/runner/mutation_executor/mutation_runner.rb +31 -12
  180. data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +1 -0
  181. data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +6 -20
  182. data/lib/evilution/runner/mutation_executor/probe_first.rb +99 -0
  183. data/lib/evilution/runner/mutation_executor/probe_first_run.rb +59 -0
  184. data/lib/evilution/runner/mutation_executor/result_cache.rb +45 -14
  185. data/lib/evilution/runner/mutation_executor/result_notifier.rb +1 -1
  186. data/lib/evilution/runner/mutation_executor/result_packer.rb +54 -24
  187. data/lib/evilution/runner/mutation_executor/runless_verdict.rb +39 -0
  188. data/lib/evilution/runner/mutation_executor/strategy/parallel/hinted_items.rb +27 -0
  189. data/lib/evilution/runner/mutation_executor/strategy/parallel/ordered_delivery.rb +97 -0
  190. data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +102 -83
  191. data/lib/evilution/runner/mutation_executor/timed_run.rb +33 -0
  192. data/lib/evilution/runner/mutation_executor/timeout_policy/adaptive.rb +67 -0
  193. data/lib/evilution/runner/mutation_executor/timeout_policy/base.rb +35 -0
  194. data/lib/evilution/runner/mutation_executor/timeout_policy/fixed.rb +17 -0
  195. data/lib/evilution/runner/mutation_executor/timeout_policy.rb +56 -0
  196. data/lib/evilution/runner/mutation_executor.rb +75 -10
  197. data/lib/evilution/runner/mutation_planner.rb +29 -21
  198. data/lib/evilution/runner/phase_clock.rb +22 -0
  199. data/lib/evilution/runner/report_publisher.rb +1 -1
  200. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  201. data/lib/evilution/runner.rb +87 -26
  202. data/lib/evilution/session/diff.rb +9 -2
  203. data/lib/evilution/session/store.rb +17 -2
  204. data/lib/evilution/source_ast_cache.rb +10 -0
  205. data/lib/evilution/spec_ast_cache.rb +15 -0
  206. data/lib/evilution/spec_resolver/constant_references.rb +66 -0
  207. data/lib/evilution/spec_resolver/project_files.rb +50 -0
  208. data/lib/evilution/spec_resolver/sibling_spec_ranker.rb +111 -0
  209. data/lib/evilution/spec_resolver/spec_pattern.rb +19 -0
  210. data/lib/evilution/spec_resolver.rb +76 -65
  211. data/lib/evilution/spec_selector.rb +17 -13
  212. data/lib/evilution/subject.rb +7 -2
  213. data/lib/evilution/version.rb +1 -1
  214. data/lib/evilution/worker_database/copies.rb +73 -0
  215. data/lib/evilution/worker_database/schema.rb +39 -0
  216. data/lib/evilution/worker_database/sqlite_copy.rb +65 -0
  217. data/lib/evilution/worker_database.rb +214 -0
  218. data/schema/evilution.config.schema.json +74 -2
  219. metadata +84 -13
  220. data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +0 -42
  221. data/scripts/benchmark_density +0 -262
  222. data/scripts/benchmark_density.yml +0 -19
  223. data/scripts/canary_manifest.yml +0 -47
  224. data/scripts/compare_mutations +0 -421
  225. data/scripts/compare_mutations.yml +0 -24
  226. data/scripts/compare_targeting +0 -279
  227. data/scripts/compare_targeting.example.yml +0 -24
  228. data/scripts/mutant_json_adapter +0 -227
@@ -0,0 +1,576 @@
1
+ # Operator yield study
2
+
3
+ Status: **study, no code change.** The default profile stays as it is; a trim
4
+ is proposed for a later decision. Decision at the end.
5
+
6
+ ## Question
7
+
8
+ 1.5.0 added 19 operators and more mutants per existing operator (the
9
+ `integer_literal` width sentinel and `n - 1`, `float_literal` `NAN` /
10
+ `INFINITY`, `array_literal` / `hash_literal` delete-each, hash key rename,
11
+ `constant_namespace_strip`, `variable_read_to_nil`, ...). Which operators, or
12
+ variants of one, cost run time without adding information, and could move to
13
+ `strict` or to a `fast` profile?
14
+
15
+ "Information" is measured the way the mutation-analysis literature does:
16
+
17
+ - **Sufficient operator sets** (Offutt, Lee, Rothermel, Untch, Zapf, *An
18
+ experimental determination of sufficient mutant operators*, TOSEM 1996): a
19
+ test set that kills the mutants of a subset of the operators kills almost
20
+ all mutants of the rest, so the rest add cost but little test-adequacy
21
+ signal.
22
+ - **Subsumption / dominator mutants** (Ammann, Delamaro, Offutt, *Establishing
23
+ theoretical minimal sets of mutants*, ICST 2014; Kurtz et al., *Mutant
24
+ subsumption graphs*, 2014): mutant A subsumes mutant B when A is killed by
25
+ at least one test and every test that kills A also kills B. B is then
26
+ redundant: any test set that kills A kills B. Kill sets measured on a real
27
+ test suite give the *dynamic* version of the relation, which is what is
28
+ measured here.
29
+
30
+ ## Method
31
+
32
+ Evilution tree: 1.5.1 plus 44 unreleased commits (commit `0d55d97`; the new
33
+ options among them, `--probe-first` and `skip_sites`, are off by default and
34
+ were not used), `--jobs 2` (Redmine `--jobs 1`), one file per run, on copies
35
+ of the `scripts/benchmark_perf.yml` corpus. 8 targets: 3 fixture files and 5
36
+ real projects.
37
+
38
+ | Target | File | Tests | Mutants |
39
+ | --- | --- | --- | ---: |
40
+ | fixture (3 targets) | `lib/pricing.rb`, `lib/inventory.rb`, `lib/grade.rb` | `spec/*_spec.rb` (RSpec) | 440 |
41
+ | dry-monads | `lib/dry/monads/maybe.rb` | `spec/unit/maybe_spec.rb` (RSpec) | 611 |
42
+ | httparty | `lib/httparty.rb` | `spec/httparty_spec.rb` (RSpec), fixture path made absolute | 1412 |
43
+ | grape | `lib/grape/declared_params_handler.rb` | `spec/grape/endpoint/declared_spec.rb` (RSpec) | 693 |
44
+ | i18n | `lib/i18n/backend/base.rb` | `test/i18n_test.rb` (Minitest) | 1837 |
45
+ | redmine | `app/models/tracker.rb` | `test/unit/tracker_test.rb` (Minitest, Rails) | 500 |
46
+ | **total** | | | **5493** |
47
+
48
+ The two known corpus artifacts are avoided as in the
49
+ [probe-first spike](probe_first_spike.md): httparty runs with
50
+ `spec/httparty_spec.rb:226` opening its fixture by an absolute path (with the
51
+ cwd-relative one every mutant is a fake kill), and dry-monads uses `maybe.rb`
52
+ instead of `list.rb` (whose class body cannot be re-evaluated, so every mutant
53
+ is killed).
54
+
55
+ Three runs per target:
56
+
57
+ 1. **Default run** (`stop_on_kill` on, as users run it): verdicts and each
58
+ mutant's `duration`. Time share is the operator's share of the summed
59
+ durations (killtime).
60
+ 2. **Kill-set run**: `--no-stop-on-kill`, so every selected example runs, with
61
+ a measurement-only hook (see [Reproducing](#reproducing)) that records the
62
+ ids of **all** failing examples per mutant. The JSON report names only the
63
+ first killer (`killing_test`), which is not enough for subsumption.
64
+ 3. **Generation dump**: every mutation the operators generate, before the
65
+ planner removes duplicates. Two operators often generate the same code;
66
+ the planner keeps one mutant and reports it under the operator registered
67
+ first. Dropping that operator does not remove the mutant: it comes back
68
+ under the next operator that generates it, with the same verdict. The
69
+ dump says which mutants come back.
70
+
71
+ Verdicts agreed between the two test runs for every mutant on every target. 58
72
+ kills (dry-monads 11, httparty 32, redmine 15) have no failing example: the
73
+ mutated file failed to load (a class-body or `def`-line mutation that raises
74
+ when the file is evaluated). They count as killed but cannot be shown
75
+ subsumed, so they are always counted as information lost when dropped.
76
+
77
+ **Test scope.** Each target runs one test file, and evilution's default
78
+ lexical example targeting narrows it further to the examples that mention the
79
+ mutated method. Subsumption is measured against that pool. With a small pool
80
+ many mutants share the same few killing examples, so dynamic subsumption
81
+ over-approximates true subsumption: a fuller suite can tell apart mutants
82
+ that look redundant here. i18n is the extreme case: 1499 of its 1837 mutants
83
+ survive (18% score), so its kill data is thin.
84
+
85
+ Definitions:
86
+
87
+ - **Kill rate**: (killed + timed out) / (killed + survived + timed out).
88
+ - **Subsumed share**: among the operator's killed mutants with a kill set, the
89
+ share that is not a dominator. A *site* (same line, or same subject =
90
+ method) is partitioned by kill set; a kill-set class with no proper subset
91
+ among the site's classes is minimal, and its members share one dominator
92
+ credit (so three identical minimal mutants are 1/3 dominator each,
93
+ 2/3 subsumed). Line-level compares the mutants of one line; subject-level is
94
+ the wider and still sound comparison (all mutants of one method run the same
95
+ selected examples, so their kill sets are comparable).
96
+ - **Drop cost** of a set of operators/variants, per target. A mutant of the
97
+ set is **removed** unless a kept operator generates the same code; then it
98
+ **returns** under that operator and costs nothing. Of the removed mutants,
99
+ the cost is the killed ones for which **no kept mutant of the same subject**
100
+ has a non-empty kill set contained in theirs ("lost kills": these are the
101
+ kills whose test requirement nothing else carries), timeouts and load-time
102
+ kills (never subsumed), and the **survivor lines lost** (lines where a
103
+ removed mutant survived and no kept mutant survives). Survivors have no kill
104
+ set, so their redundancy cannot be measured from test results; a survivor
105
+ on a line where another kept mutant also survives points the reader at the
106
+ same gap, one that is alone on its line does not.
107
+
108
+ ## Results per operator
109
+
110
+ All targets together, sorted by killtime. 106 of the 155 default operators
111
+ fired on the corpus.
112
+
113
+ | operator | mutants | share | killtime s | time share | killed | survived | timed out | other | kill rate | subsumed (line) | subsumed (subject) |
114
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
115
+ | method_call_removal | 393 | 7.2% | 60.8 | 7.4% | 235 | 158 | 0 | 0 | 59.8% | 82.4% | 91.0% |
116
+ | integer_literal | 170 | 3.1% | 51.6 | 6.3% | 105 | 65 | 0 | 0 | 61.8% | 75.0% | 82.4% |
117
+ | method_body_replacement | 310 | 5.6% | 46.3 | 5.7% | 217 | 91 | 0 | 2 | 70.5% | 71.3% | 94.9% |
118
+ | string_literal | 174 | 3.2% | 38.6 | 4.7% | 91 | 83 | 0 | 0 | 52.3% | 86.4% | 95.3% |
119
+ | symbol_literal | 328 | 6.0% | 38.3 | 4.7% | 160 | 168 | 0 | 0 | 48.8% | 85.1% | 92.8% |
120
+ | argument_nil_substitution | 302 | 5.5% | 36.7 | 4.5% | 152 | 149 | 0 | 1 | 50.5% | 86.6% | 91.7% |
121
+ | argument_list_removal | 261 | 4.8% | 35.1 | 4.3% | 151 | 110 | 0 | 0 | 57.9% | 92.0% | 96.4% |
122
+ | call_to_nil | 241 | 4.4% | 33.2 | 4.1% | 151 | 90 | 0 | 0 | 62.7% | 84.8% | 94.2% |
123
+ | statement_deletion | 197 | 3.6% | 28.2 | 3.4% | 124 | 73 | 0 | 0 | 62.9% | 84.2% | 91.9% |
124
+ | argument_removal | 217 | 4.0% | 27.5 | 3.4% | 115 | 102 | 0 | 0 | 53.0% | 89.8% | 93.7% |
125
+ | conditional_negation | 200 | 3.6% | 27.3 | 3.3% | 117 | 83 | 0 | 0 | 58.5% | 79.8% | 93.9% |
126
+ | method_body_to_raise | 153 | 2.8% | 22.6 | 2.8% | 113 | 39 | 0 | 1 | 74.3% | 85.8% | 95.5% |
127
+ | argument_propagation | 129 | 2.3% | 19.0 | 2.3% | 72 | 57 | 0 | 0 | 55.8% | 83.6% | 90.6% |
128
+ | send_mutation | 76 | 1.4% | 18.1 | 2.2% | 19 | 54 | 0 | 3 | 26.0% | 90.1% | 93.8% |
129
+ | nil_replacement | 76 | 1.4% | 17.6 | 2.2% | 44 | 32 | 0 | 0 | 57.9% | 83.8% | 91.4% |
130
+ | conditional_branch | 110 | 2.0% | 16.9 | 2.1% | 66 | 44 | 0 | 0 | 60.0% | 81.0% | 93.8% |
131
+ | argument_order_permutation | 122 | 2.2% | 14.6 | 1.8% | 58 | 64 | 0 | 0 | 47.5% | 88.9% | 92.4% |
132
+ | negation_insertion | 100 | 1.8% | 13.4 | 1.6% | 70 | 30 | 0 | 0 | 70.0% | 95.7% | 98.9% |
133
+ | variable_read_to_nil | 124 | 2.3% | 13.2 | 1.6% | 59 | 65 | 0 | 0 | 47.6% | 75.5% | 93.1% |
134
+ | boolean_operand_promotion | 88 | 1.6% | 11.5 | 1.4% | 55 | 33 | 0 | 0 | 62.5% | 69.3% | 83.2% |
135
+ | conditional_flip | 85 | 1.5% | 11.3 | 1.4% | 60 | 25 | 0 | 0 | 70.6% | 93.7% | 99.1% |
136
+ | predicate_replacement | 88 | 1.6% | 10.7 | 1.3% | 61 | 27 | 0 | 0 | 69.3% | 83.8% | 88.0% |
137
+ | comparison_replacement | 68 | 1.2% | 10.3 | 1.3% | 45 | 23 | 0 | 0 | 66.2% | 82.3% | 86.0% |
138
+ | method_body_to_super | 60 | 1.1% | 9.4 | 1.2% | 37 | 23 | 0 | 0 | 61.7% | 76.0% | 85.3% |
139
+ | array_literal | 46 | 0.8% | 9.3 | 1.1% | 14 | 32 | 0 | 0 | 30.4% | 88.9% | 93.1% |
140
+ | bang_method | 51 | 0.9% | 8.7 | 1.1% | 14 | 37 | 0 | 0 | 27.5% | 70.8% | 81.4% |
141
+ | argument_method_call_replacement | 62 | 1.1% | 8.5 | 1.0% | 22 | 40 | 0 | 0 | 35.5% | 92.7% | 97.3% |
142
+ | keyword_argument_removal | 70 | 1.3% | 8.0 | 1.0% | 36 | 34 | 0 | 0 | 51.4% | 66.2% | 99.0% |
143
+ | optional_parameter_to_required | 55 | 1.0% | 7.5 | 0.9% | 26 | 28 | 0 | 1 | 48.1% | 62.7% | 74.0% |
144
+ | optional_default_injection | 52 | 0.9% | 7.4 | 0.9% | 33 | 19 | 0 | 0 | 63.5% | 27.1% | 73.8% |
145
+ | boolean_literal_replacement | 64 | 1.2% | 7.2 | 0.9% | 16 | 48 | 0 | 0 | 25.0% | 74.5% | 86.3% |
146
+ | short_circuit_to_eager | 44 | 0.8% | 6.9 | 0.8% | 15 | 29 | 0 | 0 | 34.1% | 96.7% | 98.3% |
147
+ | collection_replacement | 37 | 0.7% | 6.7 | 0.8% | 28 | 7 | 0 | 2 | 80.0% | 88.4% | 94.8% |
148
+ | block_pass_removal | 26 | 0.5% | 5.8 | 0.7% | 5 | 21 | 0 | 0 | 19.2% | 36.5% | 37.2% |
149
+ | range_replacement | 5 | 0.1% | 5.8 | 0.7% | 0 | 5 | 0 | 0 | 0.0% | – | – |
150
+ | local_variable_assignment | 53 | 1.0% | 5.7 | 0.7% | 32 | 21 | 0 | 0 | 60.4% | 93.6% | 98.6% |
151
+ | constant_namespace_strip | 42 | 0.8% | 5.5 | 0.7% | 24 | 17 | 0 | 1 | 58.5% | 89.4% | 94.2% |
152
+ | keyword_value_swap | 39 | 0.7% | 5.5 | 0.7% | 26 | 13 | 0 | 0 | 66.7% | 71.8% | 83.1% |
153
+ | boolean_operator_replacement | 44 | 0.8% | 5.4 | 0.7% | 34 | 10 | 0 | 0 | 77.3% | 92.8% | 96.1% |
154
+ | boolean_expression_to_nil | 43 | 0.8% | 5.2 | 0.6% | 32 | 11 | 0 | 0 | 74.4% | 95.3% | 98.0% |
155
+ | index_write_to_value | 35 | 0.6% | 4.9 | 0.6% | 27 | 8 | 0 | 0 | 77.1% | 89.0% | 93.1% |
156
+ | compound_assignment | 20 | 0.4% | 4.7 | 0.6% | 17 | 3 | 0 | 0 | 85.0% | 79.7% | 98.1% |
157
+ | block_param_removal | 27 | 0.5% | 4.6 | 0.6% | 17 | 10 | 0 | 0 | 63.0% | 79.5% | 94.2% |
158
+ | index_assignment_removal | 26 | 0.5% | 3.8 | 0.5% | 19 | 7 | 0 | 0 | 73.1% | 88.3% | 93.5% |
159
+ | if_branch_swap | 22 | 0.4% | 3.7 | 0.4% | 17 | 5 | 0 | 0 | 77.3% | 93.6% | 98.1% |
160
+ | return_value_removal | 20 | 0.4% | 3.5 | 0.4% | 12 | 8 | 0 | 0 | 60.0% | 86.0% | 87.4% |
161
+ | instance_variable_write | 19 | 0.3% | 3.4 | 0.4% | 12 | 6 | 0 | 1 | 66.7% | 64.3% | 82.1% |
162
+ | index_to_fetch | 32 | 0.6% | 3.0 | 0.4% | 14 | 15 | 0 | 3 | 48.3% | 58.9% | 92.9% |
163
+ | block_removal | 21 | 0.4% | 2.9 | 0.3% | 12 | 9 | 0 | 0 | 57.1% | 95.1% | 98.6% |
164
+ | index_receiver_to_self | 32 | 0.6% | 2.8 | 0.3% | 21 | 11 | 0 | 0 | 65.6% | 93.9% | 99.0% |
165
+ | block_body_to_raise | 21 | 0.4% | 2.8 | 0.3% | 12 | 9 | 0 | 0 | 57.1% | 95.8% | 99.8% |
166
+ | attribute_write_to_read | 20 | 0.4% | 2.7 | 0.3% | 16 | 4 | 0 | 0 | 80.0% | 86.9% | 92.8% |
167
+ | keyword_argument | 12 | 0.2% | 2.7 | 0.3% | 8 | 4 | 0 | 0 | 66.7% | 92.9% | 100.0% |
168
+ | index_to_key_predicate | 30 | 0.5% | 2.6 | 0.3% | 14 | 16 | 0 | 0 | 46.7% | 89.5% | 91.6% |
169
+ | splat_operator | 16 | 0.3% | 2.4 | 0.3% | 3 | 13 | 0 | 0 | 18.8% | 50.0% | 54.2% |
170
+ | binary_operand_promotion | 13 | 0.2% | 2.3 | 0.3% | 13 | 0 | 0 | 0 | 100.0% | 77.5% | 86.6% |
171
+ | arithmetic_replacement | 13 | 0.2% | 2.2 | 0.3% | 12 | 1 | 0 | 0 | 92.3% | 86.4% | 91.8% |
172
+ | receiver_replacement | 5 | 0.1% | 2.2 | 0.3% | 4 | 1 | 0 | 0 | 80.0% | 79.5% | 85.2% |
173
+ | string_interpolation | 16 | 0.3% | 2.1 | 0.3% | 8 | 8 | 0 | 0 | 50.0% | 91.3% | 97.3% |
174
+ | return_keyword_removal | 17 | 0.3% | 2.0 | 0.2% | 12 | 5 | 0 | 0 | 70.6% | 89.1% | 92.1% |
175
+ | coercion_emptying | 8 | 0.1% | 1.9 | 0.2% | 5 | 3 | 0 | 0 | 62.5% | 87.3% | 89.8% |
176
+ | visibility_removal | 5 | 0.1% | 1.8 | 0.2% | 0 | 5 | 0 | 0 | 0.0% | – | – |
177
+ | block_body_to_nil | 10 | 0.2% | 1.8 | 0.2% | 7 | 3 | 0 | 0 | 70.0% | 50.0% | 83.3% |
178
+ | case_when | 43 | 0.8% | 1.8 | 0.2% | 7 | 36 | 0 | 0 | 16.3% | 66.7% | 98.6% |
179
+ | constant_read_to_nil | 25 | 0.5% | 1.7 | 0.2% | 5 | 20 | 0 | 0 | 20.0% | 62.3% | 72.2% |
180
+ | memoization_break | 5 | 0.1% | 1.7 | 0.2% | 0 | 5 | 0 | 0 | 0.0% | – | – |
181
+ | forwarded_argument_drop | 9 | 0.2% | 1.5 | 0.2% | 5 | 4 | 0 | 0 | 55.6% | 17.8% | 37.8% |
182
+ | mixin_removal | 9 | 0.2% | 1.4 | 0.2% | 0 | 9 | 0 | 0 | 0.0% | – | – |
183
+ | hash_literal | 12 | 0.2% | 1.4 | 0.2% | 12 | 0 | 0 | 0 | 100.0% | 92.3% | 93.8% |
184
+ | freeze_removal | 3 | 0.1% | 1.4 | 0.2% | 0 | 3 | 0 | 0 | 0.0% | – | – |
185
+ | regexp_mutation | 10 | 0.2% | 1.3 | 0.2% | 6 | 4 | 0 | 0 | 60.0% | 88.3% | 96.7% |
186
+ | superclass_removal | 9 | 0.2% | 1.3 | 0.2% | 0 | 9 | 0 | 0 | 0.0% | – | – |
187
+ | block_parameter_drop | 9 | 0.2% | 1.2 | 0.2% | 3 | 6 | 0 | 0 | 33.3% | 98.0% | 98.0% |
188
+ | constant_write_to_nil | 8 | 0.1% | 1.1 | 0.1% | 4 | 4 | 0 | 0 | 50.0% | 0.0% | 0.0% |
189
+ | index_to_at | 14 | 0.3% | 1.1 | 0.1% | 4 | 10 | 0 | 0 | 28.6% | 100.0% | 100.0% |
190
+ | integer_division_to_fdiv | 1 | 0.0% | 1.0 | 0.1% | 0 | 1 | 0 | 0 | 0.0% | – | – |
191
+ | alias_removal | 8 | 0.1% | 1.0 | 0.1% | 0 | 8 | 0 | 0 | 0.0% | – | – |
192
+ | block_body_promotion | 4 | 0.1% | 0.9 | 0.1% | 3 | 1 | 0 | 0 | 75.0% | 72.2% | 83.3% |
193
+ | equality_to_identity | 8 | 0.1% | 0.9 | 0.1% | 2 | 6 | 0 | 0 | 25.0% | 94.3% | 94.3% |
194
+ | yield_statement | 7 | 0.1% | 0.9 | 0.1% | 5 | 2 | 0 | 0 | 71.4% | 66.7% | 76.7% |
195
+ | regex_simplification | 4 | 0.1% | 0.7 | 0.1% | 2 | 2 | 0 | 0 | 50.0% | 0.0% | 50.0% |
196
+ | float_literal | 10 | 0.2% | 0.7 | 0.1% | 10 | 0 | 0 | 0 | 100.0% | 90.0% | 90.0% |
197
+ | attr_accessor_swap | 4 | 0.1% | 0.7 | 0.1% | 0 | 4 | 0 | 0 | 0.0% | – | – |
198
+ | regexp_anchor_promotion | 3 | 0.1% | 0.6 | 0.1% | 0 | 3 | 0 | 0 | 0.0% | – | – |
199
+ | dynamic_dispatch_resolution | 3 | 0.1% | 0.6 | 0.1% | 0 | 3 | 0 | 0 | 0.0% | – | – |
200
+ | comparison_operand_swap | 1 | 0.0% | 0.4 | 0.1% | 1 | 0 | 0 | 0 | 100.0% | 100.0% | 100.0% |
201
+ | regexp_anchor_to_predicate | 2 | 0.0% | 0.4 | 0.0% | 0 | 2 | 0 | 0 | 0.0% | – | – |
202
+ | block_destructuring_expansion | 3 | 0.1% | 0.3 | 0.0% | 2 | 1 | 0 | 0 | 66.7% | 83.3% | 100.0% |
203
+ | symbol_to_proc_replacement | 2 | 0.0% | 0.3 | 0.0% | 2 | 0 | 0 | 0 | 100.0% | 100.0% | 100.0% |
204
+ | explicit_super_mutation | 2 | 0.0% | 0.3 | 0.0% | 1 | 0 | 0 | 1 | 100.0% | 0.0% | 87.5% |
205
+ | dig_to_fetch_chain | 1 | 0.0% | 0.2 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
206
+ | rescue_body_replacement | 6 | 0.1% | 0.2 | 0.0% | 0 | 6 | 0 | 0 | 0.0% | – | – |
207
+ | index_range_to_drop | 1 | 0.0% | 0.2 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
208
+ | next_statement | 1 | 0.0% | 0.2 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
209
+ | regexp_option_removal | 1 | 0.0% | 0.2 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
210
+ | double_negation_removal | 1 | 0.0% | 0.2 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
211
+ | forwarding_super_to_explicit | 1 | 0.0% | 0.2 | 0.0% | 1 | 0 | 0 | 0 | 100.0% | 0.0% | 75.0% |
212
+ | rescue_class_widening | 4 | 0.1% | 0.2 | 0.0% | 0 | 4 | 0 | 0 | 0.0% | – | – |
213
+ | regexp_character_type_complement | 1 | 0.0% | 0.1 | 0.0% | 1 | 0 | 0 | 0 | 100.0% | 100.0% | 100.0% |
214
+ | typed_default_return | 2 | 0.0% | 0.1 | 0.0% | 2 | 0 | 0 | 0 | 100.0% | 50.0% | 87.5% |
215
+ | rescue_removal | 3 | 0.1% | 0.1 | 0.0% | 0 | 3 | 0 | 0 | 0.0% | – | – |
216
+ | last_expression_removal | 2 | 0.0% | 0.1 | 0.0% | 1 | 1 | 0 | 0 | 50.0% | 66.7% | 66.7% |
217
+ | regex_capture | 2 | 0.0% | 0.1 | 0.0% | 0 | 2 | 0 | 0 | 0.0% | – | – |
218
+ | regexp_alternation_branch_deletion | 2 | 0.0% | 0.1 | 0.0% | 0 | 2 | 0 | 0 | 0.0% | – | – |
219
+ | regexp_capture_to_passive | 1 | 0.0% | 0.0 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
220
+ | collection_return | 1 | 0.0% | 0.0 | 0.0% | 0 | 1 | 0 | 0 | 0.0% | – | – |
221
+ | **total** | 5493 | 100% | 817.0 | 100% | 3095 | 2382 | 0 | 16 | 56.5% | 82.4% | 92.1% |
222
+
223
+ What the table says:
224
+
225
+ - **No operator dominates the cost.** The largest, `method_call_removal`, is
226
+ 7.4% of killtime; the top ten together are 48%. Cost follows volume: per
227
+ mutant every operator costs about the same, since a mutant's time is mostly
228
+ the selected examples' time.
229
+ - **Redundancy is everywhere, not in a few operators.** 82% of kills are
230
+ subsumed at line level and 92% at subject level, with almost every operator
231
+ between 80% and 99%. This matches the literature (most mutants are
232
+ redundant; the minimal set is small), and it is a property of the test
233
+ pool (a few examples kill most mutants of a method) at least as much as of
234
+ the operators. With one test file per target, narrowed by example
235
+ targeting (see *Test scope* above), these shares are upper bounds: a full
236
+ suite could make some "subsumed" mutants distinguishing.
237
+ - **The 1.5.0 additions are not the cost.** 14 of the 19 operators added in
238
+ 1.5.0 fired on the corpus (`variable_read_to_nil`, `short_circuit_to_eager`,
239
+ `constant_namespace_strip`, `index_write_to_value`, `index_receiver_to_self`,
240
+ `index_to_key_predicate`, `constant_read_to_nil`, `constant_write_to_nil`,
241
+ `memoization_break`, `visibility_removal`, `attr_accessor_swap`,
242
+ `freeze_removal`, `rescue_class_widening`, `index_range_to_drop`); together
243
+ they produce 362 mutants (6.6%) and 5.5% of killtime. `complex_literal`,
244
+ `rational_literal`, `operator_write_receiver_promotion`, `sleep_to_zero`
245
+ and `round_half_mode_swap` did not fire. The whole `array_literal` and
246
+ `hash_literal` operators, delete-each included, are 1.1% and 0.2% of
247
+ killtime.
248
+ - **Least subsumed, most informative**: `block_pass_removal` (37%),
249
+ `optional_default_injection` (74% subject / 27% line),
250
+ `optional_parameter_to_required`, `forwarded_argument_drop`, and the
251
+ `integer_literal` neighbours `n + 1` / `n - 1` (below). These carry
252
+ requirements no other operator generates.
253
+ - **Most subsumed** (>= 98% at subject level, operators with >= 20 mutants):
254
+ `block_body_to_raise`, `conditional_flip`, `keyword_argument_removal`,
255
+ `index_receiver_to_self`, `negation_insertion`, `local_variable_assignment`,
256
+ `case_when`, `block_removal`, `short_circuit_to_eager`,
257
+ `compound_assignment`, `if_branch_swap`, `boolean_expression_to_nil`.
258
+
259
+ ### Variants
260
+
261
+ **`integer_literal` by variant**
262
+
263
+ | variant | mutants | share | killtime s | time share | killed | survived | timed out | other | kill rate | subsumed (line) | subsumed (subject) |
264
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
265
+ | width_sentinel | 41 | 24.1% | 13.8 | 26.6% | 24 | 17 | 0 | 0 | 58.5% | 82.5% | 90.2% |
266
+ | nil | 41 | 24.1% | 10.5 | 20.4% | 27 | 14 | 0 | 0 | 65.9% | 96.2% | 98.0% |
267
+ | succ | 18 | 10.6% | 8.5 | 16.4% | 10 | 8 | 0 | 0 | 55.6% | 37.7% | 42.0% |
268
+ | from_zero(+1/-1) | 24 | 14.1% | 7.7 | 14.9% | 8 | 16 | 0 | 0 | 33.3% | 80.8% | 85.2% |
269
+ | zero | 29 | 17.1% | 6.2 | 12.1% | 24 | 5 | 0 | 0 | 82.8% | 86.5% | 89.4% |
270
+ | pred | 17 | 10.0% | 4.9 | 9.6% | 12 | 5 | 0 | 0 | 70.6% | 19.9% | 51.9% |
271
+ | **total** | 170 | 100% | 51.6 | 100% | 105 | 65 | 0 | 0 | 61.8% | 75.0% | 82.4% |
272
+
273
+ The width sentinel is counted before the steps from zero: a `0` literal gets
274
+ `1`, `-1` and the sentinel `167`.
275
+
276
+ **`string_literal` by variant**
277
+
278
+ | variant | mutants | share | killtime s | time share | killed | survived | timed out | other | kill rate | subsumed (line) | subsumed (subject) |
279
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
280
+ | nil | 86 | 49.4% | 18.9 | 48.9% | 45 | 41 | 0 | 0 | 52.3% | 88.0% | 95.6% |
281
+ | empty | 58 | 33.3% | 13.0 | 33.6% | 24 | 34 | 0 | 0 | 41.4% | 88.8% | 95.9% |
282
+ | other | 30 | 17.2% | 6.8 | 17.5% | 22 | 8 | 0 | 0 | 73.3% | 80.6% | 94.3% |
283
+ | **total** | 174 | 100% | 38.6 | 100% | 91 | 83 | 0 | 0 | 52.3% | 86.4% | 95.3% |
284
+
285
+ `float_literal` produced 10 mutants on the corpus (fixture only; `NAN` 2,
286
+ `INFINITY` 4), all killed and all subsumed but one: too few to judge, and too
287
+ few to matter for cost.
288
+
289
+ - `n - 1` is 20% subsumed at line level and `n + 1` 38%, among the least
290
+ redundant mutants in the corpus (with `forwarded_argument_drop` and
291
+ `optional_default_injection`); dropping `n - 1` alone loses 8 kills nothing
292
+ else carries, `n + 1` 5. Keep both.
293
+ - The `nil` replacement of a literal is 96-98% subsumed for integers and
294
+ 88-96% for strings: replacing an operand with `nil` raises on the first use,
295
+ so any test that executes the line kills it, and the line's other mutants
296
+ are killed by a subset of the same tests. It is also largely a duplicate:
297
+ when the literal is a call argument, `argument_nil_substitution` generates
298
+ the same `nil`, so dropping the `nil` variants of `integer_literal`,
299
+ `string_literal` and `float_literal` removes only 64 of their 129 mutants
300
+ (1.2% of mutants, 2.2% of killtime). It loses no kill with a failing example
301
+ (2 load-time kills on Redmine) and no survivor line.
302
+ - The `integer_literal` width sentinel (new in 1.5.0) is 41 mutants (0.7%),
303
+ 83-90% subsumed; dropping it alone loses 1 kill (a default argument
304
+ `count = 1` on the fixture whose sentinel only one example notices). Not
305
+ worth a change either way.
306
+
307
+ ## Dropping operators
308
+
309
+ ### One at a time
310
+
311
+ Dropping any single operator loses almost nothing: for nearly every operator,
312
+ each kill is subsumed by a kept mutant of the same method, and almost every
313
+ survivor shares its line with a kept survivor. "Generated" counts the
314
+ operator's mutants in the default run; "returned" the ones another operator
315
+ generates too, which stay in the run after the drop. Selected rows (all
316
+ targets, by killtime removed):
317
+
318
+ | operator | generated | removed | returned | killtime s | time share | kills | lost kills | survivors | survivor lines lost |
319
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
320
+ | method_call_removal | 393 | 372 | 21 | 58.1 | 7.1% | 221 | 8 | 151 | 4 |
321
+ | integer_literal | 170 | 151 | 19 | 48.8 | 6.0% | 95 | 20 | 56 | 2 |
322
+ | method_body_replacement | 310 | 231 | 79 | 34.0 | 4.2% | 166 | 0 | 63 | 3 |
323
+ | symbol_literal | 328 | 258 | 70 | 30.9 | 3.8% | 121 | 13 | 137 | 0 |
324
+ | string_literal | 174 | 130 | 44 | 29.8 | 3.7% | 64 | 0 | 66 | 1 |
325
+ | statement_deletion | 197 | 197 | 0 | 28.2 | 3.4% | 124 | 5 | 73 | 2 |
326
+ | integer_literal:width_sentinel | 41 | 41 | 0 | 13.8 | 1.7% | 24 | 1 | 17 | 0 |
327
+ | negation_insertion | 100 | 100 | 0 | 13.4 | 1.6% | 70 | 0 | 30 | 0 |
328
+ | variable_read_to_nil | 124 | 124 | 0 | 13.2 | 1.6% | 59 | 0 | 65 | 5 |
329
+ | string_literal:empty | 58 | 58 | 0 | 13.0 | 1.6% | 24 | 0 | 34 | 0 |
330
+ | conditional_flip | 85 | 85 | 0 | 11.3 | 1.4% | 60 | 0 | 25 | 0 |
331
+ | predicate_replacement | 88 | 88 | 0 | 10.7 | 1.3% | 61 | 0 | 27 | 0 |
332
+ | string_literal:nil | 86 | 42 | 44 | 10.1 | 1.2% | 18 | 0 | 24 | 0 |
333
+ | conditional_negation | 200 | 88 | 112 | 9.4 | 1.2% | 57 | 0 | 31 | 0 |
334
+ | integer_literal:succ | 18 | 18 | 0 | 8.5 | 1.0% | 10 | 5 | 8 | 0 |
335
+ | keyword_argument_removal | 70 | 70 | 0 | 8.0 | 1.0% | 36 | 2 | 34 | 0 |
336
+ | integer_literal:nil | 41 | 22 | 19 | 7.7 | 0.9% | 17 | 2 | 5 | 0 |
337
+ | integer_literal:from_zero(+1/-1) | 24 | 24 | 0 | 7.7 | 0.9% | 8 | 1 | 16 | 0 |
338
+ | optional_default_injection | 52 | 52 | 0 | 7.4 | 0.9% | 33 | 6 | 19 | 1 |
339
+ | collection_replacement | 37 | 37 | 0 | 6.7 | 0.8% | 28 | 0 | 7 | 0 |
340
+ | integer_literal:zero | 29 | 29 | 0 | 6.2 | 0.8% | 24 | 1 | 5 | 0 |
341
+ | block_pass_removal | 26 | 26 | 0 | 5.8 | 0.7% | 5 | 3 | 21 | 2 |
342
+ | conditional_branch | 110 | 44 | 66 | 5.7 | 0.7% | 27 | 0 | 17 | 0 |
343
+ | constant_namespace_strip | 42 | 42 | 0 | 5.5 | 0.7% | 24 | 0 | 17 | 0 |
344
+ | boolean_operator_replacement | 44 | 44 | 0 | 5.4 | 0.7% | 34 | 0 | 10 | 0 |
345
+ | integer_literal:pred | 17 | 17 | 0 | 4.9 | 0.6% | 12 | 8 | 5 | 0 |
346
+ | boolean_expression_to_nil | 43 | 38 | 5 | 4.5 | 0.6% | 27 | 0 | 11 | 0 |
347
+ | float_literal:infinity | 4 | 4 | 0 | 0.3 | 0.0% | 4 | 0 | 0 | 0 |
348
+ | float_literal:nan | 2 | 2 | 0 | 0.1 | 0.0% | 2 | 0 | 0 | 0 |
349
+ | argument_method_call_replacement | 62 | 0 | 62 | 0.0 | 0.0% | 0 | 0 | 0 | 0 |
350
+
351
+ `argument_method_call_replacement` (`foo(bar.baz)` -> `foo(bar)`) is a pure
352
+ duplicate on this corpus: `method_call_removal` generates every one of its 62
353
+ mutants, so dropping it changes nothing. `conditional_negation` replaces an
354
+ `if` / `unless` condition with `true` and `false`; when the condition is a
355
+ `?` call, `predicate_replacement` generates the same code, so 112 of its 200
356
+ mutants stay when it alone is dropped.
357
+
358
+ ### Together: redundancy is mutual
359
+
360
+ Single-operator redundancy does not add up. Operators subsume each other, so
361
+ the set of "redundant on its own" operators is not redundant as a set. A
362
+ leave-one-target-out check shows it: on the five projects other than Redmine
363
+ (the three fixture files are one project), hold one out; on the other four
364
+ pick every operator or literal variant that removes >= 20 mutants and whose
365
+ drop-alone cost is 0 lost kills and 0 survivor lines lost; then drop the
366
+ picked set together on the held-out project:
367
+
368
+ | held out | picked | removed on held-out | time share | kills | lost kills | survivors | survivor lines lost |
369
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
370
+ | fixture | 18 | 161 / 440 | 50.6% | 136 | 38 | 25 | 2 |
371
+ | dry-monads | 19 | 161 / 611 | 26.7% | 113 | 3 | 47 | 1 |
372
+ | httparty | 19 | 472 / 1412 | 32.3% | 365 | 24 | 107 | 3 |
373
+ | grape | 18 | 189 / 693 | 26.8% | 177 | 0 | 11 | 0 |
374
+ | i18n | 16 | 410 / 1837 | 22.0% | 93 | 0 | 316 | 9 |
375
+
376
+ The greedy pick loses 38 real kills on the fixture and 24 on httparty that
377
+ the training projects said were free. A trim has to be chosen as a set and
378
+ validated as a set.
379
+
380
+ ### Proposed trim
381
+
382
+ **Set B** is what the rule above picks in **every** fold of that table (an
383
+ operator pick covering its variants), with no hand edit. It is derived from
384
+ the five projects' data only; Redmine's data is not used, so Redmine is the
385
+ held-out check:
386
+
387
+ - `negation_insertion` (prefixes `!` to a `?` method call, `x.empty?` ->
388
+ `!x.empty?`);
389
+ - `conditional_flip` (turns `if` into `unless` and back);
390
+ - `conditional_negation` (replaces an `if` / `unless` condition with `true`
391
+ and with `false`);
392
+ - `predicate_replacement` (replaces a `?` method call with `true` and with
393
+ `false`);
394
+ - `conditional_branch` (replaces the body of one branch of an `if` / `unless`
395
+ with `nil`);
396
+ - `boolean_operator_replacement` (`&&` <-> `||`);
397
+ - `keyword_argument_removal` (drops one `key: value` argument from a call);
398
+ - the `""` variant of `string_literal` (the operator name and its other
399
+ variants stay).
400
+
401
+ None of these is dropped because of what it emits: each is in the set because
402
+ its mutants were measured redundant (subsumed by a kept mutant of the same
403
+ method, or generated by a kept operator too) on every fold. For
404
+ `boolean_operator_replacement` the measurement agrees with the logic-mutant
405
+ subsumption hierarchy, which predicts the connector mutants are subsumed by
406
+ operand mutants such as `boolean_operand_promotion`'s `a && b` -> `a` / `b`
407
+ (Kaminski, Ammann, Offutt, *Better predicate testing*, AST 2011).
408
+
409
+ Dropped jointly on all 8 targets, set B removes 689 mutants (12.5%) and
410
+ 11.6% of killtime, and loses 3 kills: 2 Redmine load-time crashes (below) and
411
+ 1 httparty kill with a failing example, `parser(Parser) if true`
412
+ (`httparty.rb:281`, `conditional_negation`), that no kept mutant of
413
+ `HTTParty::ClassMethods#format` carries once all the condition operators go
414
+ together. Set B also removes every mutant that replaces an `if` condition with
415
+ `true` or `false`.
416
+
417
+ **Set B'** = set B without `conditional_negation`, the one member whose joint
418
+ drop loses a kill on a training project. Simulated on the default runs (each
419
+ removed mutant's measured duration taken out; mutants another operator
420
+ generates stay):
421
+
422
+ | target | mutants | removed | returned | killtime s | time share | kills | lost kills | survivors | survivor lines lost | score before | score after |
423
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
424
+ | fx-pricing | 135 | 2 | 2 | 0.1 | 0.5% | 2 | 0 | 0 | 0 | 90.4% | 90.2% |
425
+ | fx-inventory | 177 | 5 | 3 | 0.3 | 1.4% | 5 | 0 | 0 | 0 | 92.7% | 92.4% |
426
+ | fx-grade | 128 | 3 | 1 | 0.3 | 0.7% | 3 | 0 | 0 | 0 | 82.8% | 82.4% |
427
+ | dm-maybe | 611 | 39 | 16 | 5.9 | 6.9% | 32 | 0 | 7 | 0 | 66.3% | 65.2% |
428
+ | hp | 1412 | 68 | 11 | 10.8 | 4.9% | 63 | 0 | 5 | 0 | 71.9% | 70.8% |
429
+ | grape | 693 | 127 | 6 | 24.1 | 17.9% | 121 | 0 | 6 | 0 | 87.7% | 86.0% |
430
+ | i18n | 1837 | 201 | 17 | 7.9 | 10.9% | 56 | 0 | 145 | 0 | 18.1% | 16.9% |
431
+ | redmine | 500 | 44 | 10 | 18.0 | 8.2% | 30 | 2 | 14 | 0 | 70.1% | 70.3% |
432
+ | **total** | 5493 | 489 (8.9%) | 66 | 67.4 | 8.2% | 312 | 2 | 177 | 0 | | |
433
+
434
+ On every target, every kill set B' removes is subsumed by a kept mutant of the
435
+ same method, except 2 Redmine kills that are load-time crashes: dropping the
436
+ `:maximum` keyword from `validates_length_of :name, :maximum => ...` (and the
437
+ same for `:description`) raises when the model loads, which any test loading
438
+ the model notices. On Redmine, the held-out project, set B' loses no kill with
439
+ a failing example and no survivor line. (With Redmine in the training data
440
+ the folds no longer pick `keyword_argument_removal`: exactly the cost
441
+ Redmine shows.)
442
+
443
+ ### Measured wall time
444
+
445
+ Set B' was measured by running the default set and set B' alternately on the
446
+ same tree (set B' removed through the same hook, `DROP_OPERATORS` /
447
+ `DROP_VARIANTS`), two repeats each, Redmine one, on six targets (the fixture
448
+ through `lib/grade.rb` only). The machine was shared with other jobs during
449
+ these runs, so wall times are noisy: repeats of the same run differ by up to
450
+ 14% (httparty 116-132 s).
451
+
452
+ | target | mutants off | mutants on | Δ mutants | killtime off s | killtime on s | Δ killtime | wall off s (mean) | wall on s (mean) | Δ wall | killed mutants removed | score off | score on |
453
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
454
+ | fx-grade | 128 | 125 | -2.3% | 39.6 | 39.4 | -0.6% | 21.2 (20.4, 22.0) | 20.3 (20.2, 20.3) | -4.3% | 3 | 82.8% | 82.4% |
455
+ | dm-maybe | 611 | 572 | -6.4% | 122.0 | 108.4 | -11.2% | 57.8 (60.2, 55.4) | 49.2 (53.3, 45.1) | -14.8% | 41 | 66.3% | 65.2% |
456
+ | grape | 693 | 566 | -18.3% | 152.3 | 119.9 | -21.3% | 75.2 (75.4, 75.0) | 61.4 (59.2, 63.6) | -18.3% | 123 | 87.7% | 86.0% |
457
+ | i18n | 1837 | 1636 | -10.9% | 89.0 | 84.7 | -4.9% | 45.0 (45.3, 44.8) | 39.8 (42.3, 37.4) | -11.5% | 57 | 18.1% | 16.9% |
458
+ | hp | 1412 | 1344 | -4.8% | 269.3 | 270.7 | +0.5% | 124.0 (131.7, 116.3) | 119.2 (131.7, 106.7) | -3.9% | 66 | 71.9% | 70.8% |
459
+ | redmine | 500 | 456 | -8.8% | 245.4 | 210.9 | -14.1% | 239.1 | 204.0 | -14.7% | 35 | 70.1% | 70.3% |
460
+ | **total** | 5181 | 4699 | -9.3% | 917.7 | 833.8 | -9.1% | 562.3 | 493.9 | -12.2% | 325 | | |
461
+
462
+ "Killed mutants removed" counts killed mutants whose diff no longer appears
463
+ in the set-B' run (by file, line and diff, whatever operator reports it).
464
+
465
+ The mutant counts and scores match the simulation exactly on every target
466
+ (482 of 5181 mutants, 9.3%, over these six targets), because the simulation
467
+ lets a dropped mutant return when a kept operator generates the same code.
468
+ The returning ones, by file, line and diff between the two runs: of the 103
469
+ `conditional_branch` mutants in the default runs, 22 come back under
470
+ `call_to_nil` (a branch whose body is one call: replacing the body with `nil`
471
+ and the call with `nil` is the same code) and 9 under other operators; every
472
+ mutant of the other six operators and of the `""` variant is removed. (The
473
+ same holds for `argument_method_call_replacement`: in an A/B run that also
474
+ dropped it, all 60 of its mutants came back under `method_call_removal`.)
475
+
476
+ Measured, set B' removes **9.3% of mutants and 9.1% of killtime; wall time
477
+ falls 12% summed over the targets, from 4% (fixture, httparty) to 18%
478
+ (grape)**, against 8.7% of killtime simulated over the same targets. The
479
+ wall-time figures run ahead of killtime on dry-monads, i18n and Redmine, and
480
+ httparty's killtime does not move at all: that is the noise of the shared
481
+ machine, which two repeats (one on Redmine) cannot resolve. Mutant count and
482
+ killtime are the reliable figures: about 9% each. Mutation scores move by
483
+ -1.7 to +0.2 points.
484
+
485
+
486
+ ## Decision
487
+
488
+ - **Default profile: no change now; set B' is the recommended trim, for the
489
+ maintainer to decide.** The numbers for that call: set B' takes out 9% of
490
+ mutants and 8% of killtime (simulated; measured on six targets: 9.3% of
491
+ mutants, 9.1% of killtime, 12% of summed wall time), up to
492
+ 18% of killtime on a target heavy in conditions (grape). On 8 targets (3
493
+ fixture files and 5 real projects) it loses no kill with a failing example
494
+ that another mutant does not carry (2 load-time crash kills on Redmine) and
495
+ no survivor line. Set B (the selection rule's output, which also drops
496
+ `conditional_negation`) takes out 12.5% of mutants and 11.6% of killtime
497
+ for 1 more lost kill on httparty, and leaves no operator that replaces an
498
+ `if` condition with `true` / `false`. Before moving anything, re-run the
499
+ kill-set measurement ([Reproducing](#reproducing)) on a wider corpus of real
500
+ projects and fuller test suites: the set was validated on one held-out
501
+ project, and subsumption measured on one test file per target is an upper
502
+ bound.
503
+ - **Versioning.** `docs/versioning.md` names *adding* an operator to `default`
504
+ a MINOR change; its MAJOR row covers "changing semantics", and it states
505
+ that mutation score values are not contract, operator names are, and the
506
+ exact mutated source an operator emits is not. Set B' has two parts that
507
+ fall differently:
508
+ - *Operator level* (`negation_insertion`, `conditional_flip`,
509
+ `predicate_replacement`, `conditional_branch`,
510
+ `boolean_operator_replacement`, `keyword_argument_removal`): moving them
511
+ from `default` to `strict` keeps every operator name, but changes what
512
+ `profile: default` means. By symmetry with the MINOR clause for adding
513
+ one, removing one is plausibly MAJOR ("changing semantics"), unless the
514
+ maintainer reads the profile's operator list as score-only (not
515
+ contract), which would make it MINOR.
516
+ - *Variant level* (the `""` replacement of `string_literal`): there is no
517
+ mechanism for it today. Profiles select whole operator classes
518
+ (`Mutator::Registry.for_profile`); the study removed the variant with a
519
+ test hook filtering `Registry#mutations_for`. Two ways to ship it:
520
+ delete the variant from `StringLiteral` outright (the operator name stays
521
+ and only its emitted mutants change, which the policy treats as not
522
+ contract: PATCH or MINOR), or keep it under `strict`, which needs either
523
+ a new operator name for it (a contract addition: MINOR) or a per-profile
524
+ operator option (new internal mechanism). The chosen mechanism decides the
525
+ versioning question for this part.
526
+ - **`fast` profile: not shipped.** A saving of about 9% does not justify a new
527
+ public profile name that must keep its meaning for the whole 1.x line. It
528
+ is smaller than `skip_uncovered` / `--probe-first`, which remove mutants no
529
+ test reaches (up to 48% of wall time on i18n, see the
530
+ [probe-first spike](probe_first_spike.md)) instead of mutants a test does
531
+ reach. If a wider corpus confirms set B', the cheaper route is the
532
+ default-profile change above; a `fast` profile would only be worth it with
533
+ a larger cut that accepts lost kills, which this data does not support.
534
+ - **Not proposed**: moving any 1.5.0 operator. They are 6.6% of mutants, and
535
+ their subsumption is no higher than the old operators'. The
536
+ `integer_literal` `n + 1` / `n - 1` neighbours are among the least
537
+ redundant mutants in the corpus. Dropping `argument_method_call_replacement`
538
+ or the `nil` replacement of literals is not proposed either: the first is a
539
+ pure duplicate of `method_call_removal` here (dropping it changes nothing),
540
+ the second mostly a duplicate of `argument_nil_substitution` (1.2% of
541
+ mutants removed).
542
+ - **Noticed on the way, not changed**: `string_literal` turns a word of a
543
+ `%w()` array into `""` and into `nil`, which inside `%w` are just two other
544
+ words (`%w(a "" c)`, `%w(a nil c)`): two mutants with the same effect per
545
+ word (Redmine `tracker.rb:28`, 12 mutants).
546
+
547
+ ## Reproducing
548
+
549
+ The hook, the variant classifier, the run script and the analysis are in
550
+ [`scripts/operator_yield/`](../scripts/operator_yield/README.md)
551
+ (measurement-only: not loaded by the gem, not packaged with it). The README
552
+ there lists the corpus setup and the commands that produce every table
553
+ above.
554
+
555
+ - **Kill-set hook** (`killsets_hook.rb`), loaded with `bundle exec ruby
556
+ -r<hook> exe/evilution run ...`. With `KILLSETS_OUT=<file.jsonl>` it
557
+ prepends `run_tests(mutation)` on `Evilution::Integration::RSpec` and
558
+ `::Minitest` (resets a per-run list, then writes one JSON line per
559
+ mutation: file, line, operator, subject, diff, failing ids) and collects the
560
+ failing ids in `RSpec#build_result`
561
+ (`Integration::RSpec::ExampleIds.failed(::RSpec.world)`) and
562
+ `Minitest#run_minitest` (its `failed_ids`); run it with
563
+ `--no-stop-on-kill`. With `GENERATED_OUT=<file.jsonl>` it writes every
564
+ mutation `MutationPlanner` generates before de-duplication, with a digest
565
+ of the mutated file. With `DROP_OPERATORS=a,b` it removes operators from
566
+ `Mutator::Registry.default`, and with `DROP_VARIANTS=op:variant,...` it
567
+ filters the mutations `Registry#mutations_for` returns by the variant
568
+ classifier.
569
+ - **Variant classifier** (`variants.rb`): the changed token of the diff's
570
+ first `-`/`+` pair, widened to whole tokens; for `integer_literal` compared
571
+ numerically (width sentinel first, then `zero`, `n + 1`, `n - 1`, `±1` from
572
+ `0`; `nil`).
573
+ - **Analysis** (`analyze.rb`, `ab_compare.rb`): matches the default run, the
574
+ kill-set records and the generation dump by file, line, operator and diff,
575
+ then computes the tables above (subsumption with fractional dominator
576
+ credit, drop cost with returning duplicates, leave-one-target-out).