saboteur 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +215 -0
  3. data/README.md +44 -6
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +116 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/presets.rb +41 -1
  20. data/lib/saboteur/cli/preview_command.rb +26 -0
  21. data/lib/saboteur/cli/run_budget_options.rb +6 -1
  22. data/lib/saboteur/cli/run_command.rb +33 -0
  23. data/lib/saboteur/cli/run_output.rb +38 -0
  24. data/lib/saboteur/cli/run_reporting.rb +6 -5
  25. data/lib/saboteur/cli/run_validation.rb +54 -0
  26. data/lib/saboteur/cli/source_set.rb +0 -18
  27. data/lib/saboteur/cli.rb +29 -305
  28. data/lib/saboteur/config.rb +76 -7
  29. data/lib/saboteur/config_file.rb +12 -8
  30. data/lib/saboteur/constant_subject_finder.rb +3 -0
  31. data/lib/saboteur/custom_map.rb +24 -5
  32. data/lib/saboteur/deep_source.rb +49 -0
  33. data/lib/saboteur/dirty_tree_warning.rb +6 -4
  34. data/lib/saboteur/doctor.rb +10 -0
  35. data/lib/saboteur/engine/baseline_guard.rb +118 -15
  36. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  37. data/lib/saboteur/engine/boot.rb +11 -3
  38. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  39. data/lib/saboteur/engine/candidate_selection.rb +16 -5
  40. data/lib/saboteur/engine/candidates.rb +4 -1
  41. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  42. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  43. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  44. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  45. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  46. data/lib/saboteur/engine/coverage_store.rb +296 -21
  47. data/lib/saboteur/engine/diagnostics.rb +24 -7
  48. data/lib/saboteur/engine/distribution.rb +2 -2
  49. data/lib/saboteur/engine/gated_run.rb +4 -1
  50. data/lib/saboteur/engine/load_coverage.rb +72 -8
  51. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  52. data/lib/saboteur/engine/partial_merge.rb +40 -0
  53. data/lib/saboteur/engine/pipelines.rb +11 -3
  54. data/lib/saboteur/engine/reference_specs.rb +71 -0
  55. data/lib/saboteur/engine/result_cache.rb +68 -5
  56. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  57. data/lib/saboteur/engine/run_budget.rb +10 -7
  58. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  59. data/lib/saboteur/engine/sandbox_policy.rb +26 -5
  60. data/lib/saboteur/engine/scheduler.rb +33 -25
  61. data/lib/saboteur/engine/selection_announcement.rb +9 -3
  62. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  63. data/lib/saboteur/engine/shadow_run.rb +38 -0
  64. data/lib/saboteur/engine/subject_selection.rb +144 -64
  65. data/lib/saboteur/engine/suite_partition.rb +22 -0
  66. data/lib/saboteur/engine/test_guess.rb +186 -32
  67. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  68. data/lib/saboteur/engine.rb +44 -11
  69. data/lib/saboteur/environment_hints.rb +55 -0
  70. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  71. data/lib/saboteur/execution/external_runner.rb +123 -1
  72. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  73. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  74. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  75. data/lib/saboteur/execution/process_table.rb +7 -3
  76. data/lib/saboteur/execution/reevaluator.rb +7 -0
  77. data/lib/saboteur/execution/sandbox.rb +39 -20
  78. data/lib/saboteur/execution/seatbelt.rb +10 -3
  79. data/lib/saboteur/execution/supervisor.rb +42 -8
  80. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  81. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  82. data/lib/saboteur/execution/worker_pool.rb +48 -4
  83. data/lib/saboteur/frameworks/rails.rb +2 -5
  84. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  85. data/lib/saboteur/gates.rb +121 -28
  86. data/lib/saboteur/install.rb +6 -1
  87. data/lib/saboteur/integration/base.rb +19 -2
  88. data/lib/saboteur/integration/rspec.rb +145 -18
  89. data/lib/saboteur/line_hits.rb +81 -0
  90. data/lib/saboteur/method_calls.rb +91 -0
  91. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  92. data/lib/saboteur/mutation_source/builtin.rb +33 -21
  93. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  94. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  95. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  96. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  97. data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
  98. data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
  99. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  100. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  101. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  102. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  103. data/lib/saboteur/preview.rb +13 -0
  104. data/lib/saboteur/refinement_activations.rb +22 -10
  105. data/lib/saboteur/reporter/agent.rb +2 -2
  106. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  107. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  108. data/lib/saboteur/reporter/review_comments.rb +151 -0
  109. data/lib/saboteur/reporter/safe_out.rb +32 -0
  110. data/lib/saboteur/reporter/stryker.rb +8 -6
  111. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  112. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  113. data/lib/saboteur/reporter/terminal.rb +45 -5
  114. data/lib/saboteur/result.rb +14 -4
  115. data/lib/saboteur/selection/coverage_map.rb +150 -9
  116. data/lib/saboteur/selection/external_test_map.rb +4 -0
  117. data/lib/saboteur/selection/git_diff.rb +24 -2
  118. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  119. data/lib/saboteur/selection/recorder.rb +52 -22
  120. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  121. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  122. data/lib/saboteur/subject.rb +16 -0
  123. data/lib/saboteur/subject_finder.rb +91 -6
  124. data/lib/saboteur/text.rb +31 -0
  125. data/lib/saboteur/version.rb +1 -1
  126. data/lib/saboteur/working_tree.rb +7 -2
  127. data/lib/saboteur.rb +9 -0
  128. metadata +48 -16
@@ -64,7 +64,7 @@ module Saboteur
64
64
  results = run_candidates(candidates) do |mutation, text|
65
65
  reuse(plan, mutation, text) || execute(plan, mutation, text)
66
66
  end
67
- checked(plan, results)
67
+ checked(plan, results, truncated: results.size < candidates.size)
68
68
  end
69
69
 
70
70
  # The per-mutant timeout: timeout_factor times what the tests took
@@ -81,14 +81,21 @@ module Saboteur
81
81
 
82
82
  # Verdicts that did not hold up are warned about once per subject;
83
83
  # only checked verdicts are stored for --incremental.
84
- # A spent budget admits no check: forking the unchanged run with no
85
- # time left would turn genuine kills into fabricated errors and cache
86
- # them. Fresh kills stand unverified instead - outside the score,
87
- # never cached - and the distribution marks the subject so the run
88
- # reports budget-partial.
89
- def checked(plan, results)
90
- return results.map { |result| unverifiable(result) } if budget_spent?
91
-
84
+ # A spent budget admits no check for a subject that finished its
85
+ # mutants: forking the unchanged run with no time left would turn
86
+ # genuine kills into fabricated errors and cache them. Fresh kills
87
+ # stand unverified instead - outside the score, never cached - and
88
+ # the distribution marks the subject so the run reports
89
+ # budget-partial.
90
+ # A subject the budget stopped mid-list is different (#588): its
91
+ # completed mutants were already measured, so their checks still
92
+ # run, uncapped - the budget stops new mutation work, not the
93
+ # confirmation of work already done. The overshoot stays bounded by
94
+ # the subject's distinct killing sequences.
95
+ def checked(plan, results, truncated: false)
96
+ return results.map { |result| unverifiable(result) } if budget_spent? && !truncated
97
+
98
+ @confirming_truncation = truncated
92
99
  subject = plan.subject
93
100
  check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
94
101
  baseline_timeout: @config["baseline_timeout"])
@@ -101,6 +108,8 @@ module Saboteur
101
108
  @diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
102
109
  end
103
110
  final
111
+ ensure
112
+ @confirming_truncation = false
104
113
  end
105
114
 
106
115
  # A kill rejected because its covering examples fail with the method
@@ -111,10 +120,12 @@ module Saboteur
111
120
  # mutant's run on the same examples. Otherwise the error stays.
112
121
  #
113
122
  # Cost bound: at most two extra forks per rejected kill, the unchanged
114
- # prefix run and one mutant run. Both spend --max-duration like every
115
- # other fork: neither starts once the budget is spent, and neither gets
116
- # more than the time left. A retry the budget stops is unverified, not
117
- # an error or a kill. Keep this bound if the retry ever grows.
123
+ # prefix run and one mutant run. Ordinarily both spend --max-duration
124
+ # like every other fork: neither starts once the budget is spent, and
125
+ # neither gets more than the time left. A retry the budget stops is
126
+ # unverified, not an error or a kill. The exception is a truncated
127
+ # subject (#588): its completed kills still retry, at the full
128
+ # timeout. Keep this bound if the retry ever grows.
118
129
  def retried_in_defined_order(plan, result)
119
130
  return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
120
131
 
@@ -129,10 +140,10 @@ module Saboteur
129
140
  end
130
141
 
131
142
  def rerun_in_prefix(plan, result, tests)
132
- return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent?
143
+ return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent? && !@confirming_truncation
133
144
 
134
145
  full = timeout_for(tests, plan.slowdown)
135
- capped = budget_capped(full)
146
+ capped = @confirming_truncation ? full : budget_capped(full)
136
147
  outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
137
148
  timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
138
149
  return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
@@ -150,11 +161,11 @@ module Saboteur
150
161
  # the check never got its answer (#296).
151
162
  def unchanged_runner(subject, plan)
152
163
  lambda do |tests, timeout|
153
- if budget_spent?
164
+ if budget_spent? && !@confirming_truncation
154
165
  Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
155
166
  message: "the --max-duration budget is spent")
156
167
  else
157
- capped = budget_capped(timeout)
168
+ capped = @confirming_truncation ? timeout : budget_capped(timeout)
158
169
  outcome = @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: capped,
159
170
  eval_prefix: eval_prefix(plan))
160
171
  note_leftovers(subject, "verdict check", outcome)
@@ -238,17 +249,14 @@ module Saboteur
238
249
  [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
239
250
  end
240
251
 
241
- # What a subject no test ran gets. Code that only ran while the app
242
- # loaded is not "no coverage" - it ran - but no forked test can re-run
243
- # it, so it gets its own status, out of the score, with a hint.
252
+ # What a subject no test ran gets. Load-only code is not "no coverage":
253
+ # it ran, but no forked test can re-run it, so it gets its own status.
244
254
  def unloadable(candidates, subject, line_numbers = nil)
255
+ return skip_all(candidates, subject, Selection::CoverageMap::PRELOADED) if @map.preloaded?(subject.file)
245
256
  return skip_all(candidates, subject, "no coverage") unless
246
257
  @map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
247
258
 
248
- candidates.map do |mutation, text|
249
- MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
250
- reason: Selection::CoverageMap::RAN_AT_LOAD)
251
- end
259
+ load_time_results(candidates, subject)
252
260
  end
253
261
 
254
262
  # A def-time macro's only callers ran while the app loaded: the
@@ -329,7 +337,7 @@ module Saboteur
329
337
  # Each mutant is checked separately: changing a body can change which
330
338
  # definition a test reaches, so one unreachable verdict cannot skip later work.
331
339
  def note_unreachable(plan, outcome)
332
- @diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
340
+ @diagnostics.warn("#{plan.subject.id}: a mutant never ran and is not scored\n #{outcome.message}")
333
341
  end
334
342
 
335
343
  # The text a known def-time wrapper needs re-evaluated with the def.
@@ -10,10 +10,15 @@ module Saboteur
10
10
  module SelectionAnnouncement
11
11
  private
12
12
 
13
+ # A --since diff with no changed source file (a spec-only or docs-only
14
+ # change) yields zero subjects, so the suite never runs: say there is
15
+ # nothing to test rather than promise a whole-suite run that does not
16
+ # happen (#607).
13
17
  def selection_message(guess)
14
- if guess.changed.zero?
15
- target = config["since"] ? "no changed source file" : "no source file"
16
- "inferred specs: #{target} to infer a spec for, so the whole suite runs"
18
+ if guess.changed.zero? && config["since"]
19
+ "inferred specs: no changed source file to infer a spec for, so there is nothing to test"
20
+ elsif guess.changed.zero?
21
+ "inferred specs: no source file to infer a spec for, so the whole suite runs"
17
22
  elsif guess.specs.empty?
18
23
  guess.no_specs_message(since: config["since"])
19
24
  else
@@ -47,6 +52,7 @@ module Saboteur
47
52
  # opt-out and would warn.
48
53
  @config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
49
54
  @guessed_tests = guess.specs
55
+ @guessed_referenced = guess.referenced
50
56
  guess.scope_message(since: config["since"])
51
57
  end
52
58
  end
@@ -0,0 +1,121 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Compares the map a partial rebuild produced with the full map (#46,
7
+ # step 3). The full map is the one a run uses; this only says whether the
8
+ # partial path would have been right.
9
+ #
10
+ # Two things are compared, and both must match:
11
+ # - the semantic evidence: which tests exist and, per source file, which
12
+ # tests hit which lines, methods and branch arms, and which lines ran
13
+ # while the app loaded. Durations, build time and the order of ids are
14
+ # left out, as are entries with no test.
15
+ # - what the result cache would watch: for every test the source files
16
+ # it touched, plus the boot files and the loaded files nothing
17
+ # observed. Evidence that matches line by line but yields another
18
+ # dependency set is worse than a line mismatch, since the set decides
19
+ # whether an old verdict is reused.
20
+ module ShadowComparison
21
+ MAX_DIFFERENCES = 50
22
+ KINDS = %i[lines methods branches].freeze
23
+
24
+ module_function
25
+
26
+ # { result: :match | :superset | :mismatch, match: bool, total: Integer, extra: Integer,
27
+ # gained: Integer,
28
+ # differences: [{kind:, name:}] } with at most MAX_DIFFERENCES entries,
29
+ # sorted by kind and name.
30
+ #
31
+ # :superset means the partial map credits tests the full map does not,
32
+ # on lines the full map also credits, and never lacks a credit the
33
+ # full map has. Anything else that differs is :mismatch, including an
34
+ # extra credit on a line the full map calls uncovered: it would turn
35
+ # "no test covers this" into "covered". Those are counted apart, in
36
+ # gained. extra counts the other extra credits (one per file, line and
37
+ # test) in lines, methods and branches.
38
+ def call(partial, full, fallback_files: [])
39
+ maps = [partial, full]
40
+ found = evidence_differences(maps) + dependency_differences(maps, fallback_files)
41
+ counted = found.select { |item| KINDS.map(&:to_s).include?(item[:kind]) }
42
+ extra = counted.sum { |item| item[:extra] }
43
+ gained = counted.sum { |item| item[:gained] }
44
+ result = if found.empty? then :match
45
+ elsif gained.positive? || found.any? { |item| item[:missing] } then :mismatch
46
+ else :superset
47
+ end
48
+ shown = found.sort_by { |item| [item[:kind], item[:name]] }.first(MAX_DIFFERENCES)
49
+ { result: result, match: result == :match, total: found.size, extra: extra, gained: gained,
50
+ differences: shown.map { |item| item.slice(:kind, :name) } }
51
+ end
52
+
53
+ def evidence_differences(maps)
54
+ partial, full = maps
55
+ found = []
56
+ unless partial.tests.sort == full.tests.sort
57
+ found << { kind: "tests", name: "the set of test ids", missing: true }
58
+ end
59
+ KINDS.each { |kind| found.concat(hit_differences(kind.to_s, hits(partial, kind), hits(full, kind))) }
60
+ found.concat(exact_differences("load", load_hits(partial), load_hits(full)))
61
+ end
62
+
63
+ # Per name: the credits only the partial map has (extra) and the ones
64
+ # only the full map has (missing).
65
+ def hit_differences(kind, left, right)
66
+ (left.keys | right.keys).filter_map do |name|
67
+ extra, gained, missing = credits(left[name] || {}, right[name] || {})
68
+ next if extra.zero? && gained.zero? && !missing
69
+
70
+ { kind: kind, name: name, extra: extra, gained: gained, missing: missing }
71
+ end
72
+ end
73
+
74
+ def credits(left, right)
75
+ extra = gained = 0
76
+ missing = false
77
+ (left.keys | right.keys).each do |line|
78
+ mine = left[line] || []
79
+ theirs = right[line] || []
80
+ (theirs.empty? ? gained += mine.size : extra += (mine - theirs).size)
81
+ missing ||= (theirs - mine).any?
82
+ end
83
+ [extra, gained, missing]
84
+ end
85
+
86
+ def exact_differences(kind, left, right)
87
+ (left.keys | right.keys).reject { |name| left[name] == right[name] }
88
+ .map { |name| { kind: kind, name: name, missing: true } }
89
+ end
90
+
91
+ # {file => {line-or-span => sorted ids}} without empty entries.
92
+ def hits(map, kind)
93
+ map.public_send(kind).each_with_object({}) do |(file, by_line), acc|
94
+ kept = by_line.transform_values(&:sort).reject { |_line, ids| ids.empty? }
95
+ acc[file] = kept unless kept.empty?
96
+ end
97
+ end
98
+
99
+ def load_hits(map)
100
+ [map.load_lines, map.load_methods].flat_map(&:to_a).each_with_object({}) do |(file, lines), acc|
101
+ acc[file] = (acc.fetch(file, []) | lines).sort unless lines.empty?
102
+ end
103
+ end
104
+
105
+ # The files every verdict depends on must match exactly. What each
106
+ # test depends on follows from its line and method credits, which are
107
+ # compared above, so it is not compared again.
108
+ def dependency_differences(maps, fallback_files)
109
+ partial, full = maps
110
+ left = dependencies(partial, fallback_files)
111
+ right = dependencies(full, fallback_files)
112
+ left.keys.reject { |name| left[name] == right[name] }.map { |name| { kind: "deps", name: name, missing: true } }
113
+ end
114
+
115
+ def dependencies(map, fallback_files)
116
+ unobserved = (fallback_files | map.loaded_files).select { |file| map.empty_for?([file]) }.sort
117
+ { "(boot files)" => map.boot_files, "(unobserved files)" => unobserved }
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,38 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # One shadow comparison (#46, step 3): would a partial rebuild from the
7
+ # last map have matched the full map this run just built? The run uses
8
+ # the full map either way; this only produces the report.
9
+ #
10
+ # The report always carries the share threshold and, when the partial
11
+ # path was possible, the observed stale share, so the constant can be
12
+ # fitted from data later.
13
+ module ShadowRun
14
+ module_function
15
+
16
+ # base: the earlier inputs snapshot and its map. current: the inputs
17
+ # snapshot of this run. rerun.call(stale_spec_files, seed) builds a map
18
+ # over just those files. clean turns a name into a line-safe string.
19
+ def call(base:, base_map:, current:, full_map:, invalidation:, fallback_files:, rerun:, clean:)
20
+ report = { "threshold" => PartialInvalidation::THRESHOLD, "base_map" => base["map_key"] }
21
+ stale = invalidation.stale_specs(base, current)
22
+ return report.merge("result" => "skipped", "reason" => "a full rebuild is needed") if stale == :full
23
+
24
+ merged = stale.empty? ? base_map : PartialMerge.call(base_map, rerun.call(stale, base["seed"]), stale)
25
+ verdict = ShadowComparison.call(merged, full_map, fallback_files: fallback_files)
26
+ report.merge("result" => verdict[:result].to_s, "extra_credits" => verdict[:extra],
27
+ "gained_credits" => verdict[:gained],
28
+ "stale" => stale.map(&clean),
29
+ "stale_share" => invalidation.share(stale), "total" => verdict[:total],
30
+ "differences" => verdict[:differences].map do |item|
31
+ item.to_h do |key, value|
32
+ [key.to_s, clean.call(value)]
33
+ end
34
+ end)
35
+ end
36
+ end
37
+ end
38
+ end
@@ -4,12 +4,39 @@
4
4
  require_relative "../text"
5
5
  require_relative "../constant_subject"
6
6
  require_relative "../constant_subject_finder"
7
+ require_relative "../attribute_finder"
7
8
  require_relative "../constant_reads"
8
9
 
9
10
  module Saboteur
10
11
  class Engine
11
12
  # The subjects to mutate: every def in the included files, narrowed by
12
13
  # --subject patterns and by --since (methods with a changed line).
14
+ # A heredoc body is code the static subject walk cannot see: methods
15
+ # defined by class_eval'ing the string, or behavior living in a
16
+ # template constant, exist only when the string is evaluated (#467).
17
+ module HeredocSpans
18
+ module_function
19
+
20
+ def changed_lines_inside?(file, changed_lines)
21
+ result = Prism.parse(File.binread(file))
22
+ return false if result.failure?
23
+
24
+ spans = collect(result.value)
25
+ changed_lines.any? { |line| spans.any? { |span| span.cover?(line) } }
26
+ end
27
+
28
+ def collect(node, spans = [])
29
+ if (node.is_a?(Prism::StringNode) || node.is_a?(Prism::InterpolatedStringNode)) &&
30
+ node.opening_loc&.slice&.start_with?("<<")
31
+ # The node location of an interpolated string stops at the opening
32
+ # marker; opening through closing covers the whole heredoc body.
33
+ spans << (node.opening_loc.start_line..node.closing_loc.end_line)
34
+ end
35
+ node.compact_child_nodes.each { |child| collect(child, spans) }
36
+ spans
37
+ end
38
+ end
39
+
13
40
  class SubjectSelection
14
41
  # Lines changed since the --since ref, by file; nil without --since.
15
42
  attr_reader :changed_lines
@@ -33,12 +60,36 @@ module Saboteur
33
60
  subjects += constant_subjects
34
61
  end
35
62
  @discovered = subjects
63
+ warn_unmatched_whole_file(subjects) if @config["since"]
36
64
  subjects = by_pattern(subjects, @config["subjects"]) if @config["subjects"].any?
37
65
  subjects = by_change(subjects, @config["since"]) if @config["since"]
38
66
  subjects = drop_intentionally_untested(subjects)
67
+ inform_nothing_to_test(subjects)
39
68
  attach_consumers(subjects)
40
69
  end
41
70
 
71
+ # A --since run can legitimately select nothing: the range is empty
72
+ # (a checkout AT the base, #643) or its changes carry no mutable Ruby
73
+ # (a docs-only diff, #659). Exit 0 with a notice, not a silent green:
74
+ # name the base, the changed-file count and why nothing was selected.
75
+ def inform_nothing_to_test(subjects)
76
+ ref = @config["since"]
77
+ return unless ref && subjects.empty?
78
+
79
+ changed = Selection::GitDiff.changed_paths(ref, root: @root)
80
+ count = changed[:present].size + changed[:deleted].size
81
+ reason = if count.zero?
82
+ "the resolved range has no changes"
83
+ elsif !@raw_changed_files.intersect?(@files)
84
+ "#{count} changed file#{'s' unless count == 1}, none in mutation scope " \
85
+ "(docs-only, excluded, or non-Ruby)"
86
+ else
87
+ "no subject covers the changed lines (layout-only edits, deletions, " \
88
+ "or intentionally untested code)"
89
+ end
90
+ @diagnostics.inform("--since #{ref}: nothing to test - #{reason}")
91
+ end
92
+
42
93
  # "Klass#meth", "Klass.meth", or a namespace prefix ("Klass", "A::B").
43
94
  def self.match?(id, pattern)
44
95
  id == pattern || %w[# . ::].any? { |separator| id.start_with?("#{pattern}#{separator}") }
@@ -83,7 +134,8 @@ module Saboteur
83
134
  else
84
135
  @diagnostics.record_discovery_failure(file, reason: "invalid_file_syntax")
85
136
  parsed.errors.first(1).each do |message|
86
- @diagnostics.warn("#{file.delete_prefix("#{@root}/")} does not parse: #{message}")
137
+ @diagnostics.warn("fix the syntax in #{Text.display_safe(file.delete_prefix("#{@root}/"))}: " \
138
+ "does not parse (#{Text.display_safe(message)})")
87
139
  end
88
140
  end
89
141
  end
@@ -96,16 +148,12 @@ module Saboteur
96
148
  subjects.any? { |subject| self.class.match?(subject.id, pattern) }
97
149
  end
98
150
  missed.each do |pattern|
99
- @diagnostics.warn("--subject #{pattern} matches no subject (patterns are names like " \
100
- "Klass, Klass#meth or Klass.meth, not globs)")
151
+ @diagnostics.warn(missed_pattern_message(pattern))
101
152
  end
102
153
  # A total miss is a typo: refuse instead of a green zero-subject
103
154
  # summary (#379). A partial miss runs, but the matched count is
104
155
  # on the record.
105
- if missed.size == patterns.size
106
- raise ConfigError, "--subject #{patterns.join(', ')} matches no subject (patterns are names like " \
107
- "Klass, Klass#meth or Klass.meth, not globs)"
108
- end
156
+ raise ConfigError, total_miss_message(missed) if missed.size == patterns.size
109
157
 
110
158
  if missed.any?
111
159
  @diagnostics.warn("#{patterns.size - missed.size} of #{patterns.size} --subject patterns matched")
@@ -113,6 +161,37 @@ module Saboteur
113
161
  subjects.select { |subject| patterns.any? { |pattern| self.class.match?(subject.id, pattern) } }
114
162
  end
115
163
 
164
+ # The refusal when every pattern missed (#379). A pattern naming an
165
+ # attribute-macro method gets the real reason - the method exists but
166
+ # has no body to mutate (#373) - instead of the pattern-format hint.
167
+ def total_miss_message(missed)
168
+ if missed.all? { |pattern| attribute_macro_for(pattern) }
169
+ return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
170
+ end
171
+
172
+ "--subject #{missed.join(', ')} matches no subject (patterns are names like " \
173
+ "Klass, Klass#meth or Klass.meth, not globs)"
174
+ end
175
+
176
+ # Why one missed pattern matched nothing: when it names a method an
177
+ # attribute macro generated, say so (#373); otherwise the pattern
178
+ # itself is suspect.
179
+ def missed_pattern_message(pattern)
180
+ macro = attribute_macro_for(pattern)
181
+ return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
182
+
183
+ "fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
184
+ "Klass, Klass#meth or Klass.meth, not globs)"
185
+ end
186
+
187
+ # The attribute macro that generated the method a pattern names, if
188
+ # any. The files are parsed for macros once, only when a pattern
189
+ # missed.
190
+ def attribute_macro_for(pattern)
191
+ attributes = @attributes ||= @files.flat_map { |file| AttributeFinder.new.call(file) }
192
+ attributes.find { |found| found.id == pattern }&.macro
193
+ end
194
+
116
195
  def by_change(subjects, ref)
117
196
  changed = changed_subjects(subjects)
118
197
  warn_changed_files_without_subjects
@@ -188,12 +267,14 @@ module Saboteur
188
267
  return subjects if reasons.empty?
189
268
 
190
269
  constant_warned_files << canonical
191
- path = file.delete_prefix("#{@root}/")
270
+ path = Text.display_safe(file.delete_prefix("#{@root}/"))
192
271
  listed = reasons.first(2).join(", ")
193
- @diagnostics.warn("--since: constant subject(s) in #{path} are not mutated: the file runs code at " \
194
- "load (#{listed}) that every constant mutant would run again, and saboteur " \
195
- "cannot vet what re-running it does. List the constant in c.whole_file to " \
196
- "replay the file anyway, or move the literal into its own file.")
272
+ listed += ", and #{reasons.size - 2} more" if reasons.size > 2
273
+ @diagnostics.warn("#{path}: list the constant in c.whole_file to replay it anyway, or move the " \
274
+ "literal into its own file\n --since: constant subject(s) in this file are not " \
275
+ "mutated: it runs code at load (#{listed}) that every constant mutant would run " \
276
+ "again, and saboteur cannot vet what re-running it does. " \
277
+ "#{whole_file_id_hint(rest)}")
197
278
  opted_in
198
279
  end
199
280
 
@@ -203,10 +284,36 @@ module Saboteur
203
284
  Array(@config["whole_file"]).any? { |pattern| File.fnmatch?(pattern.to_s, subject.id) }
204
285
  end
205
286
 
287
+ # A bare constant name only ever matches a top-level constant's id -
288
+ # never a nested one - so name the exact ids to copy into c.whole_file.
289
+ # The glob example carries its blast radius: it opts in every matching
290
+ # subject project-wide, so the exact id(s) are the safer pick (#635).
291
+ def whole_file_id_hint(subjects)
292
+ ids = subjects.map(&:id)
293
+ named = ids.first(3).map { |id| "\"#{id}\"" }.join(", ")
294
+ named += ", +#{ids.size - 3} more" if ids.size > 3
295
+ "Exact id(s): #{named} (preferred); a glob like \"*#{ids.first.split('::').last}*\" also " \
296
+ "works but opts in every matching subject project-wide."
297
+ end
298
+
299
+ # A whole_file pattern that matches nothing silently does nothing (#635):
300
+ # name the pattern and what subject ids look like, so a bare constant
301
+ # name for a nested constant (which never matches its qualified id)
302
+ # stands out. Scoped to this run: a pattern for a file outside the
303
+ # current include set also matches nothing.
304
+ def warn_unmatched_whole_file(subjects)
305
+ @config["whole_file"].each do |pattern|
306
+ next if subjects.any? { |subject| File.fnmatch?(pattern.to_s, subject.id) }
307
+
308
+ @diagnostics.warn("fix the whole_file entry \"#{Text.display_safe(pattern.to_s)}\": it matches no " \
309
+ "discovered subject in this run (subject ids look like RULES, A::B::C or A::B#m)")
310
+ end
311
+ end
312
+
206
313
  def warn_unsupported_constant(found)
207
- path = found.file.delete_prefix("#{@root}/")
208
- @diagnostics.warn("--since: changed constant #{found.id} in #{path} is not mutated: #{found.reason}. " \
209
- "Only hash and array literal entries are mutated.")
314
+ path = Text.display_safe(found.file.delete_prefix("#{@root}/"))
315
+ @diagnostics.warn("--since: changed constant #{found.id} in #{path} is not mutated\n " \
316
+ "#{found.reason}; only hash and array literal entries are mutated")
210
317
  end
211
318
 
212
319
  # The tests that can observe a constant mutant are the ones covering a
@@ -225,9 +332,9 @@ module Saboteur
225
332
  subject.consumer_defs = reads.transform_values(&:defs)
226
333
  if subject.consumer_lines.empty? && reads.any? { |_file, r| r.load.any? }
227
334
  path = subject.file.delete_prefix("#{@root}/")
228
- @diagnostics.warn("--since: changed constant #{subject.id} in #{path} is not mutated: its only " \
229
- "reads capture the value at load time in another file, so no test can " \
230
- "observe a change to it.")
335
+ @diagnostics.warn("--since: changed constant #{subject.id} in #{path} is not mutated\n " \
336
+ "its only reads capture the value at load time in another file, so no test " \
337
+ "can observe a change to it")
231
338
  next
232
339
  end
233
340
  subject
@@ -290,11 +397,10 @@ module Saboteur
290
397
  unmatched = (@changed_lines.keys.select { |file| file.end_with?(".rb") } & included) - matched
291
398
  return if unmatched.empty?
292
399
 
293
- names = unmatched.first(5).map { |file| file.delete_prefix("#{@root}/") }.join(", ")
294
- more = unmatched.size > 5 ? ", ..." : ""
400
+ names = Text.name_list(unmatched.map { |file| file.delete_prefix("#{@root}/") })
295
401
  note = constant_note(unmatched)
296
402
  @diagnostics.warn("--since: #{unmatched.size} changed Ruby file(s) have no subjects covering " \
297
- "the changed lines, so nothing is mutated from them: #{names}#{more}. " \
403
+ "the changed lines, so nothing is mutated from them: #{names}\n " \
298
404
  "#{subjectless_hint(unmatched)}#{note}")
299
405
  end
300
406
 
@@ -305,7 +411,7 @@ module Saboteur
305
411
  constants = unmatched.count { |file| constant_warned_files.include?(file) }
306
412
  return "" if constants.zero?
307
413
 
308
- " #{constants} of these #{constants == 1 ? 'file has' : 'files have'} changes inside " \
414
+ "\n #{constants} of these #{constants == 1 ? 'file has' : 'files have'} changes inside " \
309
415
  "a constant literal that is not mutated (the constant warnings above name them)."
310
416
  end
311
417
 
@@ -318,37 +424,15 @@ module Saboteur
318
424
  embedded = unmatched.select { |file| changed_lines_inside_heredoc?(file) }
319
425
  return "The file may not parse, or every change may sit outside a method." if embedded.empty?
320
426
 
321
- names = embedded.first(5).map { |file| file.delete_prefix("#{@root}/") }.join(", ")
322
- more = embedded.size > 5 ? ", ..." : ""
323
- "In #{names}#{more} the changed lines are code embedded in a string (a heredoc, often the " \
427
+ names = Text.name_list(embedded.map { |file| file.delete_prefix("#{@root}/") })
428
+ "In #{names}, the changed lines are code embedded in a string (a heredoc, often the " \
324
429
  "argument to class_eval/module_eval or a template constant); methods defined by evaluating " \
325
430
  "strings are not mutation subjects, so this run cannot see that behavior."
326
431
  end
327
432
 
328
- # A heredoc body is code the static subject walk cannot see: methods
329
- # defined by class_eval'ing the string, or behavior living in a
330
- # template constant, exist only when the string is evaluated (#467).
331
- # Changed lines inside one deserve the specific warning, not the
332
- # generic parse/method guess. A file that does not parse keeps the
333
- # generic hint: its problem is named there already.
334
- def changed_lines_inside_heredoc?(file)
335
- result = Prism.parse(File.binread(file))
336
- return false if result.failure?
337
-
338
- spans = heredoc_spans(result.value)
339
- @changed_lines[file].any? { |line| spans.any? { |span| span.cover?(line) } }
340
- end
341
-
342
- def heredoc_spans(node, spans = [])
343
- if (node.is_a?(Prism::StringNode) || node.is_a?(Prism::InterpolatedStringNode)) &&
344
- node.opening_loc&.slice&.start_with?("<<")
345
- # The node location of an interpolated string stops at the opening
346
- # marker; opening through closing covers the whole heredoc body.
347
- spans << (node.opening_loc.start_line..node.closing_loc.end_line)
348
- end
349
- node.compact_child_nodes.each { |child| heredoc_spans(child, spans) }
350
- spans
351
- end
433
+ # Changed lines inside a heredoc deserve the specific warning, not the
434
+ # generic parse/method guess (#467); the scan lives in HeredocSpans.
435
+ def changed_lines_inside_heredoc?(file) = HeredocSpans.changed_lines_inside?(file, @changed_lines[file])
352
436
 
353
437
  # A changed Ruby file that an include pattern covers but c.exclude
354
438
  # removes yields zero candidates with no other sign - this repo's own
@@ -359,12 +443,11 @@ module Saboteur
359
443
  hits = excluded_territory & (@changed_lines.keys + untracked)
360
444
  return if hits.empty?
361
445
 
362
- names = hits.sort.first(5).map { |file| file.delete_prefix("#{@root}/") }.join(", ")
363
- more = hits.size > 5 ? ", ..." : ""
364
- @diagnostics.warn("--since: #{hits.size} changed Ruby file(s) are covered by c.include but removed by " \
365
- "c.exclude, so nothing is mutated from them: #{names}#{more}. Narrow or remove the " \
366
- "c.exclude entry, or pass the file as a positional source operand (configured " \
367
- "excludes are then ignored).")
446
+ names = Text.name_list(hits.sort.map { |file| file.delete_prefix("#{@root}/") })
447
+ @diagnostics.warn("narrow or remove the c.exclude entry, or pass the file as a positional source " \
448
+ "operand: #{hits.size} changed Ruby file(s) are covered by c.include but removed " \
449
+ "by c.exclude\n --since: nothing is mutated from them: #{names} (configured " \
450
+ "excludes are then ignored)")
368
451
  end
369
452
 
370
453
  # Ruby files an include pattern and an exclude pattern both match,
@@ -389,10 +472,9 @@ module Saboteur
389
472
  end
390
473
  return subjects if dropped.empty?
391
474
 
392
- names = dropped.first(5).map(&:id).join(", ")
393
- more = dropped.size > 5 ? ", ..." : ""
475
+ names = Text.name_list(dropped)
394
476
  @diagnostics.inform("#{dropped.size} subject(s) skipped: intentionally untested in " \
395
- "config/saboteur_custom_map.rb: #{names}#{more}")
477
+ "config/saboteur_custom_map.rb: #{names}")
396
478
  kept
397
479
  end
398
480
 
@@ -417,19 +499,17 @@ module Saboteur
417
499
  def report_deleted(deleted)
418
500
  return if deleted.empty?
419
501
 
420
- names = deleted.first(5).map(&:id).join(", ")
421
- more = deleted.size > 5 ? ", ..." : ""
502
+ names = Text.name_list(deleted)
422
503
  verb = deleted.one? ? "subject was" : "subjects were"
423
- @diagnostics.inform("#{deleted.size} changed #{verb} deleted, nothing to mutate: #{names}#{more}")
504
+ @diagnostics.inform("#{deleted.size} changed #{verb} deleted, nothing to mutate: #{names}")
424
505
  end
425
506
 
426
507
  def report_unchanged_syntax(subjects)
427
508
  return if subjects.empty?
428
509
 
429
- names = subjects.first(5).map(&:id).join(", ")
430
- more = subjects.size > 5 ? ", ..." : ""
510
+ names = Text.name_list(subjects)
431
511
  @diagnostics.inform("#{subjects.size} changed subject(s) differ only in comments or formatting, " \
432
- "so they are skipped: #{names}#{more}")
512
+ "so they are skipped: #{names}")
433
513
  end
434
514
 
435
515
  def canonical_path(file)