saboteur 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +280 -1
  3. data/README.md +635 -77
  4. data/exe/saboteur +6 -2
  5. data/lib/generators/saboteur/install/install_generator.rb +21 -0
  6. data/lib/saboteur/agent_skill.rb +46 -0
  7. data/lib/saboteur/baseline/acceptance.rb +218 -0
  8. data/lib/saboteur/baseline/command.rb +80 -0
  9. data/lib/saboteur/baseline/comparison.rb +105 -0
  10. data/lib/saboteur/baseline/completeness.rb +53 -0
  11. data/lib/saboteur/baseline/evidence.rb +114 -0
  12. data/lib/saboteur/baseline/identity.rb +78 -0
  13. data/lib/saboteur/baseline/manifest.rb +164 -0
  14. data/lib/saboteur/baseline/run_evidence.rb +117 -0
  15. data/lib/saboteur/baseline/workflow.rb +121 -0
  16. data/lib/saboteur/baseline_health.rb +1 -1
  17. data/lib/saboteur/cli/exit_status.rb +93 -0
  18. data/lib/saboteur/cli/run_reporting.rb +75 -0
  19. data/lib/saboteur/cli.rb +171 -63
  20. data/lib/saboteur/config.rb +101 -19
  21. data/lib/saboteur/config_file.rb +97 -0
  22. data/lib/saboteur/custom_map.rb +170 -0
  23. data/lib/saboteur/doctor.rb +57 -9
  24. data/lib/saboteur/engine/baseline_guard.rb +44 -5
  25. data/lib/saboteur/engine/boot.rb +9 -0
  26. data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
  27. data/lib/saboteur/engine/canary.rb +74 -0
  28. data/lib/saboteur/engine/candidate_selection.rb +18 -5
  29. data/lib/saboteur/engine/candidates.rb +59 -7
  30. data/lib/saboteur/engine/coverage_store.rb +53 -6
  31. data/lib/saboteur/engine/data_inputs.rb +43 -0
  32. data/lib/saboteur/engine/diagnostics.rb +176 -2
  33. data/lib/saboteur/engine/diff_scope.rb +18 -0
  34. data/lib/saboteur/engine/distribution.rb +54 -13
  35. data/lib/saboteur/engine/result_cache.rb +10 -5
  36. data/lib/saboteur/engine/run_fingerprint.rb +40 -10
  37. data/lib/saboteur/engine/sandbox_policy.rb +108 -0
  38. data/lib/saboteur/engine/scheduler.rb +99 -36
  39. data/lib/saboteur/engine/subject_selection.rb +24 -1
  40. data/lib/saboteur/engine/suite_files.rb +8 -1
  41. data/lib/saboteur/engine/test_guess.rb +52 -0
  42. data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
  43. data/lib/saboteur/engine/verdict_check.rb +7 -6
  44. data/lib/saboteur/engine.rb +257 -22
  45. data/lib/saboteur/execution/protocol.rb +2 -1
  46. data/lib/saboteur/execution/reevaluator.rb +79 -14
  47. data/lib/saboteur/execution/runner.rb +120 -17
  48. data/lib/saboteur/execution/sandbox.rb +271 -0
  49. data/lib/saboteur/execution/stale_captures.rb +43 -0
  50. data/lib/saboteur/execution/supervisor.rb +113 -9
  51. data/lib/saboteur/execution/worker_pool.rb +20 -4
  52. data/lib/saboteur/execution/wrapper.rb +45 -0
  53. data/lib/saboteur/frameworks/rails.rb +329 -13
  54. data/lib/saboteur/gates.rb +83 -0
  55. data/lib/saboteur/help.rb +63 -23
  56. data/lib/saboteur/ignore_list.rb +76 -17
  57. data/lib/saboteur/inline_suppression.rb +65 -0
  58. data/lib/saboteur/install.rb +132 -0
  59. data/lib/saboteur/integration/base.rb +1 -0
  60. data/lib/saboteur/integration/focus_guard.rb +83 -0
  61. data/lib/saboteur/integration/rspec.rb +68 -4
  62. data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
  63. data/lib/saboteur/mutation_source/builtin.rb +30 -5
  64. data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
  65. data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
  66. data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
  67. data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
  68. data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
  69. data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
  70. data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
  71. data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
  72. data/lib/saboteur/preview.rb +106 -0
  73. data/lib/saboteur/replay.rb +1 -1
  74. data/lib/saboteur/reporter/agent.rb +87 -0
  75. data/lib/saboteur/reporter/agent_clipping.rb +29 -0
  76. data/lib/saboteur/reporter/agent_commands.rb +67 -0
  77. data/lib/saboteur/reporter/agent_incomplete.rb +60 -0
  78. data/lib/saboteur/reporter/agent_summary.rb +137 -0
  79. data/lib/saboteur/reporter/findings.rb +12 -19
  80. data/lib/saboteur/reporter/history_paths.rb +38 -0
  81. data/lib/saboteur/reporter/history_record.rb +132 -0
  82. data/lib/saboteur/reporter/html.rb +61 -2
  83. data/lib/saboteur/reporter/mutation_text.rb +36 -0
  84. data/lib/saboteur/reporter/next_steps.rb +30 -0
  85. data/lib/saboteur/reporter/palette.rb +37 -0
  86. data/lib/saboteur/reporter/score_history.rb +123 -0
  87. data/lib/saboteur/reporter/score_table.rb +57 -0
  88. data/lib/saboteur/reporter/stryker.rb +25 -5
  89. data/lib/saboteur/reporter/survivor_hints.rb +68 -0
  90. data/lib/saboteur/reporter/survivor_list.rb +126 -0
  91. data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
  92. data/lib/saboteur/reporter/terminal.rb +157 -42
  93. data/lib/saboteur/reporter/test_matrix.rb +57 -0
  94. data/lib/saboteur/result.rb +86 -10
  95. data/lib/saboteur/score.rb +89 -0
  96. data/lib/saboteur/selection/coverage_map.rb +119 -13
  97. data/lib/saboteur/selection/git_diff.rb +28 -1
  98. data/lib/saboteur/selection/recorder.rb +55 -13
  99. data/lib/saboteur/subject.rb +40 -4
  100. data/lib/saboteur/subject_finder.rb +62 -12
  101. data/lib/saboteur/version.rb +1 -1
  102. data/lib/saboteur/working_tree.rb +56 -0
  103. data/lib/saboteur.rb +10 -0
  104. metadata +58 -2
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "digest"
5
5
  require "json"
6
+ require_relative "data_inputs"
6
7
 
7
8
  module Saboteur
8
9
  class Engine
@@ -11,8 +12,8 @@ module Saboteur
11
12
  class RunFingerprint
12
13
  FORMAT_VERSION = 1
13
14
  MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
14
- LOCATION_SETTINGS = %w[cache_dir json_report html_report].freeze
15
- ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra].freeze
15
+ LOCATION_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
16
+ ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
16
17
  EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
17
18
  Entry = Data.define(:path, :kind, :state, :sha256, :required)
18
19
  Change = Data.define(:path, :reason)
@@ -178,11 +179,14 @@ module Saboteur
178
179
  end
179
180
 
180
181
  def output_overlap_issues
181
- paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) }
182
- outputs = %w[json_report html_report].filter_map { |key| @config.fetch(key) }
183
- .map { |path| canonical_path(path) }
182
+ paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) } +
183
+ DataInputs.files(@root, extra: @config["fingerprint_extra"])
184
+ # Compare physical boundaries without changing declared inventory identities.
185
+ paths = paths.map { |path| canonical_path(path) }.uniq
186
+ outputs = %w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }
187
+ .map { |path| canonical_path(path) }
184
188
  issues = outputs.filter_map do |path|
185
- if path.end_with?(".rb") || paths.include?(path) || File.directory?(path)
189
+ if path.end_with?(".rb") || paths.include?(path) || File.directory?(path) || data_input?(path)
186
190
  "output overlaps input: #{display_path(path)}"
187
191
  end
188
192
  end
@@ -192,24 +196,50 @@ module Saboteur
192
196
 
193
197
  def cache_overlap_issues(paths)
194
198
  cache = canonical_path(@config["cache_dir"])
195
- protected = [@root] + %w[app lib spec].map { |name| File.join(@root, name) } + paths
196
- overlaps = protected.any? do |path|
199
+ protected = [@root] + %w[app lib spec test/fixtures].map { |name| File.join(@root, name) } + paths
200
+ inside = protected.any? do |path|
197
201
  path == cache || path.start_with?("#{cache}/") || (path != @root && cache.start_with?("#{path}/"))
198
202
  end
203
+ overlaps = inside || data_input?(cache)
199
204
  overlaps ? ["cache directory overlaps inputs: #{display_path(cache)}"] : []
200
205
  end
201
206
 
207
+ # Declared fixture, schema or config input, or a path inside a fixture tree.
208
+ def data_input?(path)
209
+ DataInputs.match?(@root, path, extra: @config["fingerprint_extra"])
210
+ end
211
+
202
212
  def inventory(issues)
203
213
  paths = project_files(issues).to_h { |path| [path, "ruby"] }
204
214
  MANIFESTS.each { |name| paths[File.join(@root, name)] = "manifest" }
215
+ data_files(issues).each { |path| paths[path] ||= "data" }
205
216
  required = merged_required_files
206
217
  required.each { |path, (kind, _)| paths[path] = kind }
207
218
  paths.map do |path, kind|
208
- needed = required.key?(path) ? required[path].last : kind != "manifest"
219
+ needed = required.key?(path) ? required[path].last : !%w[manifest data].include?(kind)
209
220
  read_entry(path, kind, needed, issues)
210
221
  end
211
222
  end
212
223
 
224
+ # Fixtures, schema dumps and database config. A pattern with no match is
225
+ # fine; a listed file that vanishes later shows up as "removed". Like Ruby
226
+ # files, data reached through a symlink (the file or a directory above it)
227
+ # is an undeclared boundary: its real path could be an output or lie outside
228
+ # the project, so the snapshot is incomplete instead of guessing.
229
+ def data_files(issues)
230
+ DataInputs.files(@root, extra: @config["fingerprint_extra"]).reject do |path|
231
+ next false unless symlinked?(path)
232
+
233
+ issues << "symlink input: #{display_path(path)}".freeze
234
+ true
235
+ end
236
+ end
237
+
238
+ def symlinked?(path)
239
+ current = @root
240
+ path.delete_prefix("#{@root}/").split("/").any? { |part| File.symlink?(current = File.join(current, part)) }
241
+ end
242
+
213
243
  def merged_required_files
214
244
  required_files.group_by(&:first).transform_values do |entries|
215
245
  roles = entries.map { |_, kind, _| kind }
@@ -240,7 +270,7 @@ module Saboteur
240
270
 
241
271
  def excluded?(path, name)
242
272
  EXCLUDED_DIRECTORIES.include?(name) || path == canonical_path(@config["cache_dir"]) ||
243
- %w[json_report html_report].filter_map { |key| @config.fetch(key) }.any? do |output|
273
+ %w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }.any? do |output|
244
274
  path == canonical_path(output)
245
275
  end
246
276
  end
@@ -0,0 +1,108 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "fileutils"
5
+ require "tmpdir"
6
+
7
+ module Saboteur
8
+ class Engine
9
+ # Decides what each mutant child's Landlock sandbox grants (#152), and
10
+ # notes once per run when the kernel cannot honor it.
11
+ class SandboxPolicy
12
+ def initialize(config:, root:, diagnostics:)
13
+ @config = config
14
+ @root = root
15
+ @diagnostics = diagnostics
16
+ end
17
+
18
+ # The third C1 layer (#152): each mutant child makes the project tree
19
+ # read-only except tmp/, the cache dir and the system temp dir. The
20
+ # claim needs Landlock ABI 3 (Linux 6.2+): earlier ABIs cannot restrict
21
+ # truncate(), so the run degrades honestly - children run with the
22
+ # guard and canary only, noted once - unless c.sandbox = "strict",
23
+ # which refuses instead.
24
+ def hook
25
+ return nil unless @config["sandbox"]
26
+
27
+ abi = sandbox_abi
28
+ return restrict_hook if abi >= 3
29
+
30
+ if @config["sandbox"] == "strict"
31
+ raise Error, "sandbox is strict but #{sandbox_gap(abi)}; refusing to run without the promised " \
32
+ "sandbox - set c.sandbox = true to run degraded, or false to opt out"
33
+ end
34
+ # Once, through the report's warnings only: warn + inform printed the
35
+ # same line twice (D's review, #178).
36
+ unless @sandbox_note_given
37
+ @sandbox_note_given = true
38
+ @diagnostics.warn("sandbox is on but #{sandbox_gap(abi)}; children run with the hazard guard and canary only")
39
+ end
40
+ nil
41
+ end
42
+
43
+ # ENOSYS/EOPNOTSUPP are a clean "no Landlock here"; any other probe
44
+ # error (EPERM under seccomp, ...) must not silently run unsandboxed.
45
+ def sandbox_abi
46
+ Execution::Sandbox.abi_version
47
+ rescue SystemCallError => e
48
+ raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); refusing to run " \
49
+ "unsandboxed - set c.sandbox = false to proceed without it"
50
+ end
51
+
52
+ def sandbox_gap(abi)
53
+ return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
54
+
55
+ "this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
56
+ end
57
+
58
+ def restrict_hook
59
+ writable = sandbox_writable_dirs
60
+ ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
61
+ end
62
+
63
+ # c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
64
+ # ...): without a grant, writes there fail with EACCES in the child.
65
+ def sandbox_writable_dirs
66
+ user_dirs = @config["sandbox_writable"].map { |dir| File.expand_path(dir, @root) }
67
+ dirs = [File.join(@root, "tmp"), @config.path("cache_dir")] + user_dirs
68
+ dirs.each { |dir| FileUtils.mkdir_p(dir) }
69
+ real = (dirs.map { |dir| File.realpath(dir) } + [File.realpath(Dir.tmpdir)]).uniq
70
+ check_sandbox_write_set(real, user_dirs.map { |dir| File.realpath(dir) })
71
+ real
72
+ end
73
+
74
+ # The effective write set - every grant realpath'd, cache_dir, the
75
+ # project tmp and the system temp dir included - must not contain the
76
+ # project root: cache_dir = ".", a tmp symlink or a checkout inside the
77
+ # system temp dir would otherwise leave the whole tree writable without
78
+ # a word. c.sandbox = "strict" refuses: strict guards the effective
79
+ # policy, not only the kernel ABI. A user grant containing another
80
+ # grant only makes the narrower one redundant, so that stays a warning.
81
+ def check_sandbox_write_set(all_dirs, user_dirs)
82
+ holder = all_dirs.find { |dir| path_contains?(dir, @root) }
83
+ if holder
84
+ message = "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
85
+ raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
86
+
87
+ @diagnostics.warn(message)
88
+ end
89
+ user_dirs.each do |dir|
90
+ (all_dirs - [dir]).each do |other|
91
+ next unless path_contains?(dir, other)
92
+
93
+ @diagnostics.warn("sandbox grant #{dir} contains the granted dir #{other}; " \
94
+ "the narrower grant is redundant")
95
+ end
96
+ end
97
+ end
98
+
99
+ # Both sides are realpath'd, so a plain "#{dir}/" prefix test would miss
100
+ # dir == "/" (every path is under it; "//" prefixes nothing).
101
+ def path_contains?(dir, path)
102
+ return true if dir == "/"
103
+
104
+ path == dir || path.start_with?("#{dir}/")
105
+ end
106
+ end
107
+ end
108
+ end
@@ -7,16 +7,20 @@ module Saboteur
7
7
  # unit of work parallel workers will share out.
8
8
  #
9
9
  # Order of business: skip what cannot be run (refinements, not loaded,
10
- # shadowed, no covering test); coarse mutants; fine mutants only if no
11
- # coarse mutant survived; then check every kill and timeout against the
12
- # unchanged method (VerdictCheck). Each mutant runs only the tests that
13
- # executed its edit, likeliest killers first, with a timeout from those
14
- # tests' recorded durations.
10
+ # shadowed, no covering test); coarse mutants; all eligible fine mutants;
11
+ # then check every kill and timeout against the unchanged method
12
+ # (VerdictCheck). Each mutant runs only the tests that executed its edit,
13
+ # likeliest killers first, with a timeout from their recorded durations.
15
14
  class Scheduler
15
+ WHOLE_FILE_HINT = "; add it to c.whole_file to re-evaluate the whole file"
16
+
16
17
  # A subject to run, with what its mutants need. killers counts the
17
18
  # tests that killed earlier mutants of this subject; slowdown is how
18
19
  # much slower than recorded its tests ran in a child (see calibrate).
19
- Plan = Struct.new(:subject, :tests, :killers, :slowdown, keyword_init: true)
20
+ # wrapper names a known def-time wrapper (Execution::Wrapper), so the
21
+ # child re-evaluates it together with the def.
22
+ Plan = Struct.new(:subject, :tests, :killers, :slowdown, :wrapper,
23
+ keyword_init: true)
20
24
 
21
25
  # Tests that killed more earlier mutants first; otherwise keep the order.
22
26
  def self.prioritize(tests, killers)
@@ -35,20 +39,24 @@ module Saboteur
35
39
  def call(subject, candidates)
36
40
  return [] if candidates.empty?
37
41
 
38
- reason = subject.skip_reason || load_problem(subject)
42
+ reason, wrapper = subject.skip_reason ? [subject.skip_reason, nil] : load_problem(subject)
39
43
  return skip_all(candidates, subject, reason) if reason
44
+ if wrapper == :sorbet && !subject.sig_source
45
+ return skip_all(candidates, subject,
46
+ "sorbet sig not found next to the def; cannot preserve runtime type checks")
47
+ end
40
48
 
41
49
  tests = @map.tests_for(subject)
42
- return skip_all(candidates, subject, "no coverage") if tests.empty?
50
+ return unloadable(candidates, subject) if tests.empty?
43
51
 
44
- plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0)
52
+ plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0, wrapper: wrapper)
45
53
  reused = all_reused(plan, candidates)
46
54
  return reused if reused
47
55
 
48
- results = run_candidates(plan, candidates) do |mutation, text|
56
+ results = run_candidates(candidates) do |mutation, text|
49
57
  reuse(plan, mutation, text) || execute(plan, mutation, text)
50
58
  end
51
- checked(subject, results)
59
+ checked(plan, results)
52
60
  end
53
61
 
54
62
  # The per-mutant timeout: timeout_factor times what the tests took
@@ -65,9 +73,11 @@ module Saboteur
65
73
 
66
74
  # Verdicts that did not hold up are warned about once per subject;
67
75
  # only checked verdicts are stored for --incremental.
68
- def checked(subject, results)
76
+ def checked(plan, results)
77
+ subject = plan.subject
69
78
  run_unchanged = lambda do |tests, timeout|
70
- @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout)
79
+ @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout,
80
+ eval_prefix: eval_prefix(plan))
71
81
  .tap { |outcome| note_leftovers(subject, "verdict check", outcome) }
72
82
  end
73
83
  check = VerdictCheck.new(run_unchanged: run_unchanged, baseline_timeout: @config["baseline_timeout"])
@@ -86,21 +96,13 @@ module Saboteur
86
96
  return nil unless @cache
87
97
 
88
98
  catch(:must_run) do
89
- run_candidates(plan, candidates) { |mutation, text| reuse(plan, mutation, text) || throw(:must_run) }
99
+ run_candidates(candidates) { |mutation, text| reuse(plan, mutation, text) || throw(:must_run) }
90
100
  end
91
101
  end
92
102
 
93
- def run_candidates(plan, candidates, &run_one)
103
+ def run_candidates(candidates, &run_one)
94
104
  coarse, fine = candidates.partition { |mutation, _| mutation.coarse }
95
105
  results = coarse.map { |mutation, text| run_one.call(mutation, text) }
96
- # A surviving coarse mutant means nothing checks the body, so the fine
97
- # mutants would only add noise. Not so when a rescue or else can
98
- # return a value of its own: fine mutants that make the body raise
99
- # can still be caught.
100
- if results.any?(&:survived?) && !plan.subject.recovers?
101
- return results + skip_all(fine, plan.subject, "coarse mutant survived", covered_by: plan.tests)
102
- end
103
-
104
106
  results + fine.map { |mutation, text| run_one.call(mutation, text) }
105
107
  end
106
108
 
@@ -109,7 +111,7 @@ module Saboteur
109
111
  return nil unless @cache
110
112
 
111
113
  tests = tests_for(plan, mutation)
112
- return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
114
+ return uncoverable(plan, mutation, text) if tests.empty?
113
115
 
114
116
  hit = @cache.lookup(plan.subject, mutation, tests)
115
117
  return nil unless hit
@@ -121,14 +123,14 @@ module Saboteur
121
123
 
122
124
  def execute(plan, mutation, text)
123
125
  tests = tests_for(plan, mutation)
124
- return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
126
+ return uncoverable(plan, mutation, text) if tests.empty?
125
127
 
126
- timeout = timeout_for(tests, plan.slowdown)
127
- outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout, mutation: mutation)
128
- outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
128
+ outcome = run_mutant(plan, mutation, text, tests)
129
+ note_unreachable(plan, outcome) if outcome.status == :unreachable
129
130
  result = MutantResult.new(subject: plan.subject, mutation: mutation, status: outcome.status,
130
131
  mutated_source: text, reason: outcome.message, runtime: outcome.runtime,
131
- killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause)
132
+ killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause,
133
+ executed: outcome.executed_ids)
132
134
  outcome.failed_ids.each { |id| plan.killers[id] += 1 }
133
135
  note_leftovers(plan.subject, "#{mutation.operator} #{result.digest}", outcome)
134
136
  @diagnostics.progress(result)
@@ -139,8 +141,12 @@ module Saboteur
139
141
  # body, so it gets every test of the method), prior killers first, then
140
142
  # fastest-first.
141
143
  def tests_for(plan, mutation)
144
+ return self.class.prioritize(plan.tests, plan.killers) if mutation.coarse
145
+
142
146
  line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
143
- tests = plan.tests & @map.tests_for_lines(plan.subject.file, line_numbers)
147
+ tests = plan.tests & @map.tests_for_edit(plan.subject, line_numbers)
148
+ branch_tests = @map.tests_for_branches(plan.subject, mutation.edits)
149
+ tests &= branch_tests if branch_tests
144
150
  self.class.prioritize(tests, plan.killers)
145
151
  end
146
152
 
@@ -148,6 +154,52 @@ module Saboteur
148
154
  [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
149
155
  end
150
156
 
157
+ # What a subject no test ran gets. Code that only ran while the app
158
+ # loaded is not "no coverage" - it ran - but no forked test can re-run
159
+ # it, so it gets its own status, out of the score, with a hint.
160
+ def unloadable(candidates, subject, line_numbers = nil)
161
+ return skip_all(candidates, subject, "no coverage") unless
162
+ @map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
163
+
164
+ candidates.map do |mutation, text|
165
+ MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
166
+ reason: Selection::CoverageMap::RAN_AT_LOAD)
167
+ end
168
+ end
169
+
170
+ # One mutation whose edited lines no test ran (see unloadable).
171
+ def uncoverable(plan, mutation, text)
172
+ line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
173
+ result, = unloadable([[mutation, text]], plan.subject, line_numbers)
174
+ return result if result.status == :ran_at_load
175
+ return result unless @map.tests_for_branches(plan.subject, mutation.edits) == []
176
+
177
+ skipped, = skip_all([[mutation, text]], plan.subject, "no coverage (branch not taken)")
178
+ skipped
179
+ end
180
+
181
+ def run_mutant(plan, mutation, text, tests)
182
+ timeout = timeout_for(tests, plan.slowdown)
183
+ outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout,
184
+ mutation: mutation, eval_prefix: eval_prefix(plan))
185
+ outcome.status == :timeout ? calibrate(plan, mutation, text, tests, timeout) : outcome
186
+ end
187
+
188
+ # Each mutant is checked separately: changing a body can change which
189
+ # definition a test reaches, so one unreachable verdict cannot skip later work.
190
+ def note_unreachable(plan, outcome)
191
+ @diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
192
+ end
193
+
194
+ # The text a known def-time wrapper needs re-evaluated with the def.
195
+ # Sorbet's sig sits on the lines right before it; without it the def
196
+ # evaluates alone and runtime type checks silently drop for the child.
197
+ def eval_prefix(plan)
198
+ return nil unless plan.wrapper == :sorbet
199
+
200
+ plan.subject.sig_source
201
+ end
202
+
151
203
  # Recorded durations come from one warm process; a fresh child can be
152
204
  # slower (lazy loading, first queries). So a timeout is checked at
153
205
  # once: the unchanged method runs the same tests with
@@ -156,7 +208,7 @@ module Saboteur
156
208
  # the measured slowdown. A timeout that remains is the mutant's own.
157
209
  def calibrate(plan, mutation, text, tests, timeout)
158
210
  unchanged = @runner.run(subject: plan.subject, text: plan.subject.source, test_ids: tests,
159
- timeout: @config["baseline_timeout"])
211
+ timeout: @config["baseline_timeout"], eval_prefix: eval_prefix(plan))
160
212
  note_leftovers(plan.subject, "timeout check", unchanged)
161
213
  return unchanged_failed(unchanged) unless unchanged.status == :survived
162
214
 
@@ -165,7 +217,8 @@ module Saboteur
165
217
  longer = deadline(unchanged.runtime)
166
218
  return Execution::Runner::Outcome.new(status: :timeout, runtime: timeout) if longer <= timeout
167
219
 
168
- @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation)
220
+ @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation,
221
+ eval_prefix: eval_prefix(plan))
169
222
  end
170
223
 
171
224
  def unchanged_failed(unchanged)
@@ -176,18 +229,28 @@ module Saboteur
176
229
  end
177
230
 
178
231
  # The method saboteur would redefine must be the one Ruby is running.
232
+ # Returns [skip reason, wrapper]: a known def-time wrapper (sorbet's
233
+ # sig is one, see Execution::Wrapper) is not a shadowing - the child
234
+ # re-evaluates it together with the def, so the check passes.
179
235
  def load_problem(subject)
236
+ return [nil, nil] if @reevaluator.whole_file?(subject)
237
+
180
238
  owner = @reevaluator.resolve_owner(subject)
181
239
  target = subject.singleton ? owner.singleton_class : owner
182
240
  method = own_method(target, subject.name)
183
- return "method not loaded" unless method
241
+ return ["method not loaded", nil] unless method
184
242
 
185
243
  file, line = method.source_location
186
- return nil if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
244
+ if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
245
+ return [nil, nil]
246
+ end
247
+
248
+ wrapper = Execution::Wrapper.around(method, file)
249
+ return [nil, wrapper] if wrapper
187
250
 
188
- "shadowed definition (runtime method at #{file}:#{line})"
251
+ ["shadowed definition (runtime method at #{file}:#{line})#{WHOLE_FILE_HINT}", nil]
189
252
  rescue Error => e
190
- "owner not loaded: #{e.message}"
253
+ ["owner not loaded: #{e.message}#{WHOLE_FILE_HINT}", nil]
191
254
  end
192
255
 
193
256
  def own_method(target, name)
@@ -33,7 +33,9 @@ module Saboteur
33
33
  def find(files)
34
34
  finder = SubjectFinder.new
35
35
  files.flat_map do |file|
36
+ InlineSuppression.new(file, File.read(file))
36
37
  parsed = finder.call(file)
38
+ @diagnostics.record_discovery_failure(file, reason: "invalid_file_syntax") unless parsed.errors.empty?
37
39
  parsed.errors.first(1).each do |message|
38
40
  @diagnostics.warn("#{file.delete_prefix("#{@root}/")} does not parse: #{message}")
39
41
  end
@@ -54,9 +56,30 @@ module Saboteur
54
56
  def by_change(subjects, ref)
55
57
  @changed_lines = Selection::GitDiff.changed_lines(ref, root: @root)
56
58
  .transform_keys { |file| Selection::CoverageMap.canonical(file) }
57
- subjects.select do |subject|
59
+ changed = subjects.select do |subject|
58
60
  @changed_lines[subject.file]&.any? { |line| subject.contains_line?(line) }
59
61
  end
62
+ drop_unchanged_syntax(changed, ref)
63
+ end
64
+
65
+ # A method whose only changes are comments, blank lines or spacing has
66
+ # nothing new to test.
67
+ def drop_unchanged_syntax(subjects, ref)
68
+ return subjects if subjects.empty?
69
+
70
+ comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(ref, root: @root), root: @root)
71
+ same, kept = subjects.partition { |subject| comparison.unchanged?(subject) }
72
+ report_unchanged_syntax(same)
73
+ kept
74
+ end
75
+
76
+ def report_unchanged_syntax(subjects)
77
+ return if subjects.empty?
78
+
79
+ names = subjects.first(5).map(&:id).join(", ")
80
+ more = subjects.size > 5 ? ", ..." : ""
81
+ @diagnostics.inform("#{subjects.size} changed method(s) differ only in comments or formatting, " \
82
+ "so they are skipped: #{names}#{more}")
60
83
  end
61
84
  end
62
85
  end
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "digest"
5
5
  require "shellwords"
6
+ require_relative "data_inputs"
6
7
 
7
8
  module Saboteur
8
9
  class Engine
@@ -22,7 +23,7 @@ module Saboteur
22
23
  end
23
24
 
24
25
  def files
25
- (shared_spec_files + boot_and_hook_files + option_files + required_by_options).uniq.sort
26
+ (shared_spec_files + boot_and_hook_files + option_files + required_by_options + data_files).uniq.sort
26
27
  end
27
28
 
28
29
  def digest
@@ -33,6 +34,12 @@ module Saboteur
33
34
 
34
35
  private
35
36
 
37
+ # Fixtures, schema dumps and database config: not Ruby, but they change
38
+ # what the tests see, so a stale cache must not survive an edit to them.
39
+ def data_files
40
+ DataInputs.files(@root, extra: @config["fingerprint_extra"])
41
+ end
42
+
36
43
  def shared_spec_files
37
44
  Dir.glob("spec/**/*.rb", base: @root).reject { |file| file.end_with?("_spec.rb") }
38
45
  .map { |file| File.join(@root, file) }
@@ -0,0 +1,52 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # --guess-tests: the specs that sit next to the changed source files by
7
+ # naming convention (lib/a/b.rb -> spec/a/b_spec.rb). A guess, not a
8
+ # coverage fact: a spec that only reaches the file through another one
9
+ # is not found. The run scopes to the guesses only when every changed
10
+ # file has one.
11
+ class TestGuess
12
+ Result = Data.define(:specs, :unmatched, :changed)
13
+
14
+ SOURCE_ROOTS = %w[lib app].freeze
15
+
16
+ def initialize(config)
17
+ @root = File.realpath(config.root)
18
+ @config = config
19
+ end
20
+
21
+ # specs: existing spec files (relative paths); unmatched: changed
22
+ # source files with no guessed spec; changed: how many source files
23
+ # changed.
24
+ def call
25
+ changed = changed_sources
26
+ guesses = changed.to_h { |file| [file, guess_for(file)] }
27
+ Result.new(specs: guesses.values.compact.uniq.sort, unmatched: guesses.select { |_, spec| spec.nil? }.keys,
28
+ changed: changed.size)
29
+ end
30
+
31
+ private
32
+
33
+ # Changed files that saboteur would mutate, relative to the root.
34
+ def changed_sources
35
+ included = @config.included_files.to_set
36
+ changed = Selection::GitDiff.changed_lines(@config["since"], root: @root).keys
37
+ changed.map { |file| Selection::CoverageMap.canonical(file) }
38
+ .select { |file| included.include?(file) }
39
+ .map { |file| file.delete_prefix("#{@root}/") }.sort
40
+ end
41
+
42
+ def guess_for(file)
43
+ stem = file.delete_suffix(".rb")
44
+ rest = stem.split("/", 2)
45
+ candidates = ["spec/#{stem}_spec.rb"]
46
+ candidates.unshift("spec/#{rest.last}_spec.rb", "spec/lib/#{rest.last}_spec.rb") if
47
+ rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
48
+ candidates.find { |path| File.file?(File.join(@root, path)) }
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,87 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "prism"
5
+ require_relative "../inline_suppression"
6
+
7
+ module Saboteur
8
+ class Engine
9
+ # Tells whether a method still has, in the working tree, the syntax it had
10
+ # at the merge base of the --since ref. A change to a comment, a blank
11
+ # line or spacing leaves it the same, so no mutant of it asks a question
12
+ # the change raised. Comparison is of the Prism tree without locations
13
+ # and comments, plus the frozen_string_literal magic comment and the
14
+ # `sig` in front of the def, which both change what runs. `__LINE__` is
15
+ # the one node whose value is its location, so it keeps its line: a
16
+ # method that moved, or one with a line added above its `__LINE__`, is
17
+ # not the same.
18
+ #
19
+ # Anything that cannot be compared says the method changed: a file or
20
+ # method that is new at the base, a base that does not parse, a method
21
+ # name defined twice in a file.
22
+ class UnchangedSyntax
23
+ def initialize(base, root:)
24
+ @base = base
25
+ @root = root
26
+ @base_subjects = {}
27
+ @current_subjects = {}
28
+ end
29
+
30
+ def unchanged?(subject)
31
+ before = base_subject(subject)
32
+ return false unless before
33
+
34
+ return false unless suppression_signature(before) == suppression_signature(subject)
35
+
36
+ before.frozen_literals == subject.frozen_literals && before.sig_source == subject.sig_source &&
37
+ self.class.digest(before.node) == self.class.digest(subject.node)
38
+ end
39
+
40
+ # The node as plain data: type and fields, never locations (except
41
+ # the line a `__LINE__` stands for).
42
+ def self.digest(value)
43
+ case value
44
+ when Prism::SourceLineNode then [value.type, value.location.start_line]
45
+ when Prism::Node
46
+ fields = value.deconstruct_keys(nil).except(:node_id)
47
+ [value.type, fields.transform_values { |field| digest(field) }]
48
+ when Array then value.map { |item| digest(item) }
49
+ when Prism::Location then nil
50
+ else value
51
+ end
52
+ end
53
+
54
+ private
55
+
56
+ def suppression_signature(subject)
57
+ InlineSuppression.new(subject.file, subject.file_source).signature(subject)
58
+ end
59
+
60
+ def base_subject(subject)
61
+ return nil unless current_subjects(subject).key?(subject.id)
62
+
63
+ base_subjects(subject.file)[subject.id]
64
+ end
65
+
66
+ # Only the names defined once: with two defs of a name, which one a
67
+ # change belongs to is a guess.
68
+ def unique_by_id(source, file)
69
+ subjects = SubjectFinder.new.call(file, source: source).subjects
70
+ counts = subjects.map(&:id).tally
71
+ subjects.select { |subject| counts[subject.id] == 1 }.to_h { |subject| [subject.id, subject] }
72
+ end
73
+
74
+ def current_subjects(subject)
75
+ file = subject.file
76
+ @current_subjects.fetch(file) { @current_subjects[file] = unique_by_id(subject.file_source, file) }
77
+ end
78
+
79
+ def base_subjects(file)
80
+ @base_subjects.fetch(file) do
81
+ text = Selection::GitDiff.file_at(@base, file.delete_prefix("#{@root}/"), root: @root)
82
+ @base_subjects[file] = text ? unique_by_id(text, file) : {}
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end