saboteur 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +657 -0
  3. data/README.md +198 -1077
  4. data/lib/saboteur/cli/exit_status.rb +32 -1
  5. data/lib/saboteur/cli/help_options.rb +2 -1
  6. data/lib/saboteur/cli/history_command.rb +135 -0
  7. data/lib/saboteur/cli/presets.rb +6 -2
  8. data/lib/saboteur/cli/selection_options.rb +31 -0
  9. data/lib/saboteur/cli/shard_options.rb +17 -0
  10. data/lib/saboteur/cli/source_set.rb +53 -0
  11. data/lib/saboteur/cli/survivors_command.rb +157 -0
  12. data/lib/saboteur/cli.rb +71 -54
  13. data/lib/saboteur/config.rb +98 -5
  14. data/lib/saboteur/config_file.rb +8 -0
  15. data/lib/saboteur/constant_reads.rb +182 -0
  16. data/lib/saboteur/constant_subject.rb +70 -0
  17. data/lib/saboteur/constant_subject_finder.rb +281 -0
  18. data/lib/saboteur/coverage_reads.rb +29 -0
  19. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  20. data/lib/saboteur/doctor.rb +9 -0
  21. data/lib/saboteur/engine/baseline_guard.rb +62 -14
  22. data/lib/saboteur/engine/boot.rb +4 -0
  23. data/lib/saboteur/engine/candidate_selection.rb +19 -0
  24. data/lib/saboteur/engine/candidates.rb +43 -3
  25. data/lib/saboteur/engine/coverage_store.rb +32 -10
  26. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  27. data/lib/saboteur/engine/diagnostics.rb +18 -0
  28. data/lib/saboteur/engine/distribution.rb +10 -0
  29. data/lib/saboteur/engine/early_stop_recording.rb +3 -2
  30. data/lib/saboteur/engine/gated_run.rb +35 -0
  31. data/lib/saboteur/engine/load_coverage.rb +45 -0
  32. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  33. data/lib/saboteur/engine/pipelines.rb +107 -0
  34. data/lib/saboteur/engine/result_cache.rb +65 -8
  35. data/lib/saboteur/engine/run_budget.rb +27 -0
  36. data/lib/saboteur/engine/sandbox_policy.rb +38 -0
  37. data/lib/saboteur/engine/scheduler.rb +109 -3
  38. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  39. data/lib/saboteur/engine/shard.rb +69 -0
  40. data/lib/saboteur/engine/subject_selection.rb +295 -20
  41. data/lib/saboteur/engine/test_guess.rb +315 -25
  42. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  43. data/lib/saboteur/engine.rb +100 -127
  44. data/lib/saboteur/execution/external_runner.rb +471 -0
  45. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  46. data/lib/saboteur/execution/peak_memory.rb +28 -0
  47. data/lib/saboteur/execution/process_table.rb +12 -0
  48. data/lib/saboteur/execution/process_tree.rb +65 -5
  49. data/lib/saboteur/execution/reevaluator.rb +68 -12
  50. data/lib/saboteur/execution/runner.rb +46 -7
  51. data/lib/saboteur/execution/sandbox.rb +15 -2
  52. data/lib/saboteur/execution/seatbelt.rb +226 -0
  53. data/lib/saboteur/execution/supervisor.rb +27 -10
  54. data/lib/saboteur/execution/worker_pool.rb +23 -1
  55. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  56. data/lib/saboteur/gates.rb +43 -2
  57. data/lib/saboteur/help.rb +60 -9
  58. data/lib/saboteur/ignore_list.rb +29 -6
  59. data/lib/saboteur/integration/rspec.rb +135 -5
  60. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  61. data/lib/saboteur/mutation_source/builtin.rb +26 -5
  62. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  63. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  64. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  65. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  66. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  67. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  68. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  69. data/lib/saboteur/mutation_source/hazard_guard.rb +108 -0
  70. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  71. data/lib/saboteur/mutation_source/method_swap.rb +148 -0
  72. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  73. data/lib/saboteur/mutation_source/operator_examples.rb +3 -1
  74. data/lib/saboteur/mutation_source/operator_registry.rb +4 -2
  75. data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
  76. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  77. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  78. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  79. data/lib/saboteur/preview.rb +120 -12
  80. data/lib/saboteur/refinement_activations.rb +60 -0
  81. data/lib/saboteur/reporter/agent.rb +9 -1
  82. data/lib/saboteur/reporter/agent_commands.rb +10 -3
  83. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  84. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  85. data/lib/saboteur/reporter/findings.rb +5 -2
  86. data/lib/saboteur/reporter/history_policy.rb +11 -0
  87. data/lib/saboteur/reporter/history_reader.rb +47 -0
  88. data/lib/saboteur/reporter/html.rb +26 -4
  89. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  90. data/lib/saboteur/reporter/stryker.rb +27 -3
  91. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  92. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  93. data/lib/saboteur/reporter/terminal.rb +40 -7
  94. data/lib/saboteur/result.rb +19 -3
  95. data/lib/saboteur/selection/coverage_map.rb +105 -10
  96. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  97. data/lib/saboteur/selection/external_test_map.rb +27 -0
  98. data/lib/saboteur/selection/git_diff.rb +93 -11
  99. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  100. data/lib/saboteur/selection/recorder.rb +22 -4
  101. data/lib/saboteur/selection/source_scope.rb +22 -0
  102. data/lib/saboteur/subject.rb +42 -7
  103. data/lib/saboteur/subject_finder.rb +173 -29
  104. data/lib/saboteur/version.rb +1 -1
  105. data/lib/saboteur/working_tree.rb +38 -15
  106. data/lib/saboteur.rb +13 -0
  107. metadata +35 -2
@@ -10,6 +10,10 @@ module Saboteur
10
10
  # A test suite that (perhaps only under a mutant) starts a saboteur run
11
11
  # would otherwise run saboteur inside a saboteur child, recursively.
12
12
  def self.refuse_recursion!
13
+ if ENV[Execution::ExternalRunner::CHILD_MARKER]
14
+ raise Error, "refusing to start: this process runs inside a saboteur external test command " \
15
+ "(#{Execution::ExternalRunner::CHILD_MARKER} is set); a nested run could race its tree"
16
+ end
13
17
  return unless ENV[Execution::Supervisor::CHILD_ENV]
14
18
 
15
19
  raise Error, "refusing to start: this process is a saboteur child " \
@@ -22,6 +22,10 @@ module Saboteur
22
22
  reason = subject.skip_reason || (skip_reason_for(subject) if @map.tests_for(subject).empty?)
23
23
  if reason
24
24
  @skipped_subjects[subject] = reason
25
+ # In a scoped run every subject is part of the change under
26
+ # review, so a skip is said up front, not only in the end-of-run
27
+ # summary (#574).
28
+ @diagnostics.announce(skip_announcement(subject, reason)) if scope
25
29
  return @selected[subject] = []
26
30
  end
27
31
 
@@ -44,6 +48,21 @@ module Saboteur
44
48
 
45
49
  private
46
50
 
51
+ # A scoped no-coverage skip names the specs the run loaded: coverage
52
+ # living in a sibling spec file is otherwise invisible, and the false
53
+ # read is "untestable" when the truth is "not loaded" (#636). Names
54
+ # at most five, like the other selection notes.
55
+ def skip_announcement(subject, reason)
56
+ line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
57
+ return line unless reason == "no coverage"
58
+
59
+ loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
60
+ more = loaded.size > 5 ? ", ..." : ""
61
+ names = loaded.empty? ? "none" : loaded.first(5).join(", ")
62
+ "#{line} - loaded specs: #{names}#{more}; " \
63
+ "other spec files may cover it, pass --test to load them"
64
+ end
65
+
47
66
  # Load-only code is not "no coverage": it ran, just never inside a
48
67
  # test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
49
68
  def skip_reason_for(subject)
@@ -10,6 +10,25 @@ module Saboteur
10
10
  # a subject into runnable candidates: applied, re-parsed, de-duplicated,
11
11
  # coarse first. Returns [[mutation, mutated_text]].
12
12
  class Candidates
13
+ # The generator for constant hash/array literal subjects (#306); the
14
+ # method operators walk a def body a constant does not have.
15
+ def self.build(config, diagnostics, hazard_guard:)
16
+ new(build_sources(config), diagnostics, hazard_guard: hazard_guard,
17
+ constant_source: build_constant_source(config))
18
+ end
19
+
20
+ # Constants are generated by the builtin operator family, so a
21
+ # config that drops "builtin" from c.sources drops constant mutation
22
+ # too: the configured sources are the boundary for every subject
23
+ # kind (#319). Custom sources never see constants.
24
+ def self.build_constant_source(config)
25
+ return nil unless config["sources"].include?("builtin")
26
+
27
+ MutationSource::ConstantLiterals.new(operators: config["operators"],
28
+ exclude_operators: config["exclude_operators"],
29
+ arid: config["arid"])
30
+ end
31
+
13
32
  def self.build_sources(config)
14
33
  root = File.realpath(config.root)
15
34
  config["sources"].map do |entry|
@@ -19,15 +38,26 @@ module Saboteur
19
38
  end
20
39
 
21
40
  file, const = entry.split(":", 2)
22
- raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName" unless const
41
+ unless const
42
+ if entry.include?("*")
43
+ raise ConfigError, "mutation source #{entry.inspect} looks like a source glob; " \
44
+ "did you mean c.include? c.sources registers custom operator sources " \
45
+ "(builtin or path.rb:ClassName)"
46
+ end
47
+ raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName"
48
+ end
23
49
 
24
50
  require File.expand_path(file, root)
25
51
  Object.const_get(const).new
26
52
  end
27
53
  end
28
54
 
29
- def initialize(sources, diagnostics, hazard_guard: nil, bytecode: BytecodeEquivalence.new)
55
+ # constant_source: the generator for ConstantSubject instances; nil
56
+ # leaves constants with no candidates (a safe empty list).
57
+ def initialize(sources, diagnostics, hazard_guard: nil, bytecode: BytecodeEquivalence.new,
58
+ constant_source: nil)
30
59
  @sources = sources
60
+ @constant_source = constant_source
31
61
  @diagnostics = diagnostics
32
62
  @hazard_guard = hazard_guard
33
63
  @bytecode = bytecode
@@ -40,7 +70,7 @@ module Saboteur
40
70
  end
41
71
 
42
72
  seen = Set[subject.source]
43
- list = @sources.flat_map { |source| proposals(source, subject) }.filter_map do |mutation|
73
+ list = sources_for(subject).flat_map { |source| proposals(source, subject) }.filter_map do |mutation|
44
74
  candidate(subject, mutation, seen)
45
75
  end
46
76
  list = suppress_inline(list, subject)
@@ -50,6 +80,15 @@ module Saboteur
50
80
 
51
81
  private
52
82
 
83
+ # Constant literals have their own generator; external custom sources
84
+ # implement the def-subject contract and never see constants (#306).
85
+ def sources_for(subject)
86
+ return @sources unless subject.is_a?(ConstantSubject)
87
+ return [] unless @constant_source
88
+
89
+ [@constant_source]
90
+ end
91
+
53
92
  def suppress_inline(list, subject)
54
93
  directives = InlineSuppression.new(subject.file, subject.file_source)
55
94
  list.reject do |mutation, _text|
@@ -95,6 +134,7 @@ module Saboteur
95
134
  seen << text
96
135
  unless parses?(text)
97
136
  @diagnostics.count_invalid_mutation
137
+ @diagnostics.warn("#{subject.id}: dropped #{mutation.operator} mutation (did not parse)")
98
138
  return record_drop(subject, mutation, "invalid_syntax")
99
139
  end
100
140
 
@@ -15,7 +15,8 @@ module Saboteur
15
15
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
16
16
  LOG = "coverage-build.log"
17
17
 
18
- def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, timeout_cap: nil)
18
+ def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
19
+ timeout_cap: nil)
19
20
  @config = config
20
21
  @integration = integration
21
22
  @hooks = hooks
@@ -23,7 +24,9 @@ module Saboteur
23
24
  @diagnostics = diagnostics
24
25
  @timeout_cap = timeout_cap
25
26
  @load_lines = load_lines
27
+ @load_methods = load_methods
26
28
  @root = File.realpath(config.root)
29
+ @root_prefixes = Selection::CoverageMap.root_prefixes(config.root)
27
30
  end
28
31
 
29
32
  # True when fetch would reuse the cached map instead of running the
@@ -49,6 +52,21 @@ module Saboteur
49
52
 
50
53
  private
51
54
 
55
+ def write_map(writer, recorder, build_seconds)
56
+ writer.write(recorder.result(build_seconds: build_seconds, loaded_files: loaded_project_files).to_json)
57
+ writer.close
58
+ end
59
+
60
+ # Every project source this child loaded, even a lazy require outside the
61
+ # include list that the recorder drops: --incremental must watch it, or a
62
+ # change there keeps stale verdicts (#353).
63
+ def loaded_project_files
64
+ $LOADED_FEATURES.filter_map do |file|
65
+ path = Text.utf8_strict(Selection::CoverageMap.canonical(file))
66
+ path if path.end_with?(".rb") && @root_prefixes.any? { |prefix| path.start_with?(prefix) }
67
+ end
68
+ end
69
+
52
70
  def cache_path(files)
53
71
  File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
54
72
  end
@@ -122,13 +140,16 @@ module Saboteur
122
140
  # The parent's load lines merge into the freshly built map: the forked
123
141
  # build child cannot inherit the parent's Coverage counters.
124
142
  def with_parent_load_lines(map)
125
- @load_lines.empty? ? map : map.with_load_lines(@load_lines)
143
+ @load_lines.empty? && @load_methods.empty? ? map : map.with_load_lines(@load_lines, @load_methods)
126
144
  end
127
145
 
128
146
  # A whole-suite build that records nothing - no line hit and no method
129
- # call - in any included file fails the run (exit 3): usually no test
130
- # in the suite exercises these files, or the setup hid the app from
131
- # Coverage (Bootsnap's compile cache is the default Rails case). An
147
+ # call - in any included file fails the run (exit 3). The recorder's map
148
+ # also holds the suite's spec files, so the check is per included file,
149
+ # not whole-map: a suite that tests nothing here still records its own
150
+ # spec code. Usually no test in the suite exercises these files, or
151
+ # the setup hid the app from Coverage (Bootsnap's compile cache is the
152
+ # default Rails case). An
132
153
  # all-endless-def file records method calls only and passes. Either way
133
154
  # a run on a truly empty map would test nothing, and exit 0 would lie.
134
155
  # With explicit tests (--test) an empty map is a legitimate outcome
@@ -137,7 +158,7 @@ module Saboteur
137
158
  # map too, so a map cached before this check cannot keep a run green,
138
159
  # and the guard runs before saving, so an empty map is never cached.
139
160
  def refuse_empty_map(map, files)
140
- return map if files.empty? || !map.empty?
161
+ return map if files.empty? || !map.empty_for?(files)
141
162
 
142
163
  if @config["tests"].empty?
143
164
  raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
@@ -175,7 +196,10 @@ module Saboteur
175
196
  $stderr.reopen($stdout)
176
197
  @hooks.run(:after_fork)
177
198
  started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
178
- recorder = Selection::Recorder.new(files)
199
+ # The suite's own files are recorded too: a spec can read a
200
+ # constant directly, and without its line hits no test maps to the
201
+ # read (#319).
202
+ recorder = Selection::Recorder.new(files + @integration.test_files)
179
203
  recorder.start
180
204
  Kernel.srand(@config["seed"])
181
205
  passed = @integration.run_all(recorder)
@@ -183,9 +207,7 @@ module Saboteur
183
207
  raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
184
208
  "with seed #{@config['seed']}."
185
209
  end
186
- build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
187
- writer.write(recorder.result(build_seconds: build_seconds).to_json)
188
- writer.close
210
+ write_map(writer, recorder, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
189
211
  rescue Exception => e # rubocop:disable Lint/RescueException
190
212
  writer.write(JSON.generate(error: "#{e.class}: #{e.message}"))
191
213
  exit!(1)
@@ -0,0 +1,43 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The examples an order-dependent spec file needs, in the order RSpec
7
+ # defines them: for each file with a covering example, every example from
8
+ # the start of the file through its last covering example. Later examples
9
+ # cannot change earlier ones, so this is the smallest run that keeps
10
+ # whatever state the earlier examples leave behind.
11
+ class DefinedOrderPrefix
12
+ ID_PATTERN = /\A(?<file>.*)\[(?<path>[\d:]+)\]\z/
13
+
14
+ # every_test: all example ids the map knows, in any order.
15
+ def initialize(every_test)
16
+ @every_test = every_test
17
+ end
18
+
19
+ # The prefix for the covering examples; nil when it adds nothing to
20
+ # what already ran (an id that is not an RSpec id included).
21
+ def for(covering)
22
+ parsed = covering.map { |id| parse(id) }
23
+ return nil if parsed.any?(&:nil?)
24
+
25
+ prefix = parsed.group_by(&:first).flat_map { |file, entries| file_prefix(file, entries.map(&:last).max) }
26
+ prefix == covering ? nil : prefix
27
+ end
28
+
29
+ private
30
+
31
+ def file_prefix(file, last_position)
32
+ in_file = @every_test.filter_map { |id| (entry = parse(id)) && entry.first == file ? entry : nil }
33
+ through = in_file.select { |entry| (entry.last <=> last_position) <= 0 }.sort_by(&:last)
34
+ through.map { |entry| "#{file}[#{entry.last.join(':')}]" }
35
+ end
36
+
37
+ def parse(id)
38
+ match = ID_PATTERN.match(id)
39
+ match && [match[:file], match[:path].split(":").map(&:to_i)]
40
+ end
41
+ end
42
+ end
43
+ end
@@ -86,12 +86,30 @@ module Saboteur
86
86
  @discovery_failures = []
87
87
  @hazard_suppressions = 0
88
88
  @inline_suppressions = []
89
+ @erb_template_skips = []
90
+ @erb_template_note_emitted = false
89
91
  end
90
92
 
91
93
  def warn(message)
92
94
  @warnings << message
93
95
  end
94
96
 
97
+ # ERB generator templates (Rails generators carry `.rb` files with ERB
98
+ # interpolation, conventionally under `lib/generators/**/templates/`)
99
+ # can never parse as plain Ruby, so they can never hold mutation
100
+ # subjects (#372). One summarized note per run replaces the per-file
101
+ # parse warning every discovery pass used to print; a real syntax
102
+ # error in a plain Ruby file still warns per file.
103
+ def record_erb_template_skip(file, root:)
104
+ relative = file.to_s.delete_prefix("#{root}/")
105
+ @erb_template_skips << relative.freeze unless @erb_template_skips.include?(relative)
106
+ return if @erb_template_note_emitted
107
+
108
+ @erb_template_note_emitted = true
109
+ warn("skipping ERB template files that cannot parse as Ruby " \
110
+ "(for example #{relative}); templates are not mutation subjects")
111
+ end
112
+
95
113
  def count_invalid_mutation
96
114
  @invalid_mutations += 1
97
115
  end
@@ -15,6 +15,16 @@ module Saboteur
15
15
  # whose fresh kills the spent budget left unchecked (#296).
16
16
  attr_reader :unreached, :truncated, :unverified
17
17
 
18
+ # The run's peak-memory record for the JSON report (#202): each
19
+ # worker's kernel peak RSS and the supervisor's own, nil when the
20
+ # platform measured neither so the report omits the section. No total:
21
+ # fork copy-on-write makes per-process RSS non-additive.
22
+ def memory_report
23
+ workers = (@pool&.worker_peaks || {}).sort.map { |number, mb| { worker: number, peakMb: mb } }
24
+ supervisor = Execution::PeakMemory.peak_mb
25
+ workers.empty? && supervisor.nil? ? nil : { workers: workers, supervisorPeakMb: supervisor }.compact
26
+ end
27
+
18
28
  # stop_after: nil, or a callable that gets one subject's results and
19
29
  # returns true to stop before the next subject (--fail-fast). It only
20
30
  # works on one process: parallel workers have already started the
@@ -10,12 +10,13 @@ module Saboteur
10
10
  private
11
11
 
12
12
  # The subjects a stop left unrun or mid-way: the verdict covers less than the selection.
13
- def record_early_stop(distribution)
13
+ def record_early_stop(distribution, planned_mutants: nil)
14
14
  return if distribution.unreached.empty? && distribution.truncated.empty? && distribution.unverified.empty?
15
15
 
16
16
  truncated = distribution.truncated.map { |subject, omitted| { subject: subject.id, omitted: omitted } }
17
17
  @early_stop = EarlyStop.new(reason: stop_reason(distribution), unreached: distribution.unreached.map(&:id),
18
- truncated: truncated, unverified: distribution.unverified.map(&:id))
18
+ truncated: truncated, unverified: distribution.unverified.map(&:id),
19
+ planned_mutants: planned_mutants)
19
20
  end
20
21
 
21
22
  # Mid-subject omissions and skipped checks are budget work: only a
@@ -0,0 +1,35 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # A threshold or mutant-count gate judges the score, so a path that
7
+ # narrows what the score covers (a partially mapped --since run, #370)
8
+ # must keep its hard stop instead of green-lighting the change.
9
+ module GatedRun
10
+ private
11
+
12
+ def gated?
13
+ config["threshold"] || Gates.new(config).any?
14
+ end
15
+
16
+ # Reports the guess's skipped and subjectless files, then stops or
17
+ # warns on unmapped files per the gate (#370).
18
+ def report_unmapped(guess)
19
+ @diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
20
+ inform_skipped_specs(guess)
21
+ stop_or_warn_unmapped(guess)
22
+ end
23
+
24
+ # A --since run with a partial mapping runs the mapped subset with a
25
+ # loud warning per unmapped file (#370); a full run, a zero guess and
26
+ # a gated run keep the hard stop.
27
+ def stop_or_warn_unmapped(guess)
28
+ return if guess.unmatched.empty?
29
+ raise ConfigError, guess.unmatched_message if !config["since"] || guess.specs.empty? || gated?
30
+
31
+ guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,45 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "coverage"
5
+
6
+ module Saboteur
7
+ class Engine
8
+ # The boot's coverage snapshot, read in the parent because a forked
9
+ # child does not inherit Coverage counters: the lines and the def
10
+ # calls that ran while the app and spec files loaded. The executable
11
+ # counts from its first line, so this snapshot holds the whole boot.
12
+ # The lines tell ran-at-load mutants apart from uncovered ones; the
13
+ # def calls name def-time macros whose only caller ran here, so their
14
+ # mutants read as ran at load instead of surviving on the pristine
15
+ # closure (#409).
16
+ module LoadCoverage
17
+ # Returns [load_lines, load_methods]: file => line numbers for each.
18
+ def self.capture(files)
19
+ return [{}, {}] unless Coverage.state == :suspended
20
+
21
+ wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
22
+ CoverageReads.result(stop: false, clear: false)
23
+ .each_with_object([{}, {}]) do |(path, data), (lines_h, methods_h)|
24
+ file = Selection::CoverageMap.canonical(path)
25
+ next unless wanted.include?(file)
26
+
27
+ lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
28
+ lines -= uncalled_shared_lines(data)
29
+ lines_h[file] = lines unless lines.empty?
30
+ defs = data.fetch(:methods, {}).filter_map { |(_owner, _name, first, *), count| first if count.positive? }
31
+ methods_h[file] = defs.uniq.sort unless defs.empty?
32
+ end
33
+ end
34
+
35
+ # A one-line or endless definition executes its def line even when its
36
+ # body is never called. If any sibling on that physical line is uncalled,
37
+ # discard its ambiguous line evidence rather than crediting every body.
38
+ def self.uncalled_shared_lines(data)
39
+ shared = data.fetch(:methods, {}).select { |(_owner, _name, first, _column, last, *), _| first == last }
40
+ calls = shared.group_by { |(_owner, _name, first, *), _count| first }
41
+ calls.filter_map { |line, entries| line if entries.any? { |_key, count| count.zero? } }
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,21 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The --since stop whose changed source files all sit outside the
7
+ # include scope (#483), kept apart from the guess flow it guards.
8
+ module OutOfScopeGuard
9
+ private
10
+
11
+ # Every changed source file sits outside the include scope (#483): the
12
+ # whole-suite fallback would run the slowest possible suite against
13
+ # zero subjects, so stop and name what the scope hid.
14
+ def refuse_out_of_scope_changes(guess)
15
+ return unless guess.changed.zero? && guess.out_of_scope.any?
16
+
17
+ raise ConfigError, guess.out_of_scope_message
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,107 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # The two run pipelines: in-process (boot, coverage, workers) and the
7
+ # --test-command external mode (#140).
8
+ module Pipelines
9
+ # The in-process pipeline: boot the target, mutate under coverage, and
10
+ # clean the workers up however the run ends.
11
+ def run_pipeline(ignore_list, started)
12
+ return call_external(ignore_list, started) if external_command?
13
+
14
+ boot = Boot.new(config, hooks)
15
+ begin
16
+ integration, files, load_lines, load_methods = boot_target(boot)
17
+ @rails = boot.rails
18
+ subjects, scope = select_subjects(files, test_files: integration.test_files)
19
+ results = if subjects.empty?
20
+ []
21
+ else
22
+ mutate(subjects, files, integration, scope, ignore_list,
23
+ load_lines:, load_methods:)
24
+ end
25
+ @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
26
+ build_report(results, subjects, scope, started)
27
+ ensure
28
+ # Cleanup runs however the call ends - a partway boot failure, no
29
+ # subjects, no runnable work - so a host embedding Engine never
30
+ # holds worker copies until process exit. Idempotent; the Rails
31
+ # profile's at_exit handler stays the fallback for abrupt exit.
32
+ begin
33
+ @coverage_guard&.resume # an embedded host gets its coverage back (#254)
34
+ ensure
35
+ safely_cleanup_workers(boot.rails)
36
+ end
37
+ end
38
+ end
39
+
40
+ # The --test-command pipeline (#140): no boot, no coverage, no
41
+ # baseline - every mutant runs the command on the mutated tree,
42
+ # serially, with no sandbox. The command is arbitrary user code.
43
+ def call_external(ignore_list, started)
44
+ @diagnostics.inform("external test command: #{config['test_command']} " \
45
+ "(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
46
+ files = config.included_files
47
+ subjects, scope = select_subjects(files)
48
+ results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
49
+ @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
50
+ build_report(results, subjects, scope, started)
51
+ end
52
+
53
+ def mutate_external(subjects, scope, ignore_list)
54
+ runner = external_runner
55
+ # One neutral run before any mutant exists: a command that fails on
56
+ # the unchanged tree can never claim kills (#305). It answers to the
57
+ # run budget with an ABSOLUTE deadline, admitted after the tree copy:
58
+ # a budget already spent is a budget stop, not a command error.
59
+ begin
60
+ runner.preflight!(deadline: preflight_deadline, budget_deadline: budget_deadline)
61
+ rescue BudgetPhaseTimeout => e
62
+ return budget_phase_stop(e, subjects)
63
+ end
64
+ map = Selection::ExternalTestMap.new
65
+ scheduler = Scheduler.new(config:, runner:, map:,
66
+ reevaluator: Execution::Reevaluator.new(config["reeval"],
67
+ whole_file: config["whole_file"]),
68
+ diagnostics: @diagnostics, deadline: budget_deadline, verify_loaded: false)
69
+ lists = candidate_lists(subjects, scope, map)
70
+ work = subjects.zip(lists).reject { |_, list| list.empty? }
71
+ return [] if work.empty?
72
+
73
+ runnable_subjects, runnable_lists = work.transpose
74
+ begin
75
+ warn_unless_mutants_visible(runner, runnable_subjects.first)
76
+ rescue BudgetPhaseTimeout => e
77
+ return budget_phase_stop(e, subjects)
78
+ end
79
+ run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
80
+ ensure
81
+ runner.close
82
+ end
83
+
84
+ # Warn loudly when the command cannot see mutants at all: an
85
+ # anchored or never-loaded file reads as 100 % survivors (#441).
86
+ def warn_unless_mutants_visible(runner, subject)
87
+ warning = runner.visibility_check(subject: subject, deadline: preflight_deadline,
88
+ budget_deadline: budget_deadline)
89
+ @diagnostics.warn(warning) if warning
90
+ end
91
+
92
+ def external_command? = !config["test_command"].to_s.empty?
93
+
94
+ # The preflight's absolute monotonic deadline: its own
95
+ # baseline_timeout slice, or the run budget's end, whichever is
96
+ # sooner. Absolute, so the tree copy before admission cannot eat the
97
+ # budget while the command still gets a full timeout (#305 Architect).
98
+ def preflight_deadline
99
+ own = Process.clock_gettime(Process::CLOCK_MONOTONIC) + config["baseline_timeout"]
100
+ deadline = budget_deadline
101
+ deadline ? [own, deadline].min : own
102
+ end
103
+
104
+ def external_runner = Execution::ExternalRunner.new(command: config["test_command"], root: @root)
105
+ end
106
+ end
107
+ end
@@ -15,25 +15,36 @@ module Saboteur
15
15
  # same: the method's source, the whole file the method lives in (so a
16
16
  # constant or helper changed elsewhere in that file is seen), the mutant,
17
17
  # the set of covering tests, the content of those tests' files, the
18
- # lockfile, the run's inputs (config, saboteur version). Only killed, survived
19
- # and timeout are stored;
18
+ # lockfile, the run's inputs (config, saboteur version), and every source
19
+ # file a covering test touched (a method it calls, a constant defined
20
+ # elsewhere - #197). Only killed, survived and timeout are stored;
20
21
  # errors and skips always run again.
21
22
  #
22
- # Known hole, the reason this is opt-in: a change in another file that
23
- # this method uses (a method it calls, a constant defined elsewhere) is
24
- # not seen, so a stored verdict may be stale after it.
23
+ # When the dependency set is unknown - no coverage map, or an empty one -
24
+ # every source file is watched instead: reuse turns conservative, never
25
+ # stale.
25
26
  class ResultCache
26
27
  FORMAT = 1
27
28
  STORED = %i[killed survived timeout].freeze
28
29
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
30
+ # Dependency declarations themselves, so the no-lockfile library
31
+ # shape (bundler's own convention: libraries gitignore the lockfile)
32
+ # still invalidates on a dependency edit (#408).
33
+ MANIFESTS = %w[Gemfile gems.rb].freeze
34
+
35
+ SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
29
36
 
30
37
  # suite_digest: SuiteFiles#digest, so an edit to a spec helper or
31
38
  # support file invalidates every verdict.
32
- def initialize(path:, root:, inputs:, suite_digest: "")
39
+ def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [])
33
40
  @path = path
34
41
  @root = root
35
42
  @inputs = inputs
36
43
  @suite_digest = suite_digest
44
+ # Engine always passes the built map; a missing one behaves as an
45
+ # empty one, which watches every fallback and child-loaded source (#197).
46
+ @coverage = coverage || Selection::CoverageMap.new(lines: {}, tests: [])
47
+ @fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
37
48
  @entries = load
38
49
  @file_digests = {}
39
50
  end
@@ -65,6 +76,16 @@ module Saboteur
65
76
 
66
77
  private
67
78
 
79
+ # The current project source files, relative and sorted. A file added
80
+ # after the map was built changes the set and invalidates every stored
81
+ # verdict, whether or not any watched file references it yet.
82
+ def source_set
83
+ @source_set ||= Dir.glob("**/*.rb", base: @root)
84
+ .reject { |rel| SOURCE_EXCLUDES.include?(rel.split("/").first) }
85
+ .sort
86
+ .join("\0")
87
+ end
88
+
68
89
  def load
69
90
  data = JSON.parse(Text.utf8_strict(File.read(@path)))
70
91
  data["format"] == FORMAT ? data["entries"] : {}
@@ -78,12 +99,33 @@ module Saboteur
78
99
 
79
100
  def fingerprint(subject, tests)
80
101
  digest = Digest::SHA256.new
81
- digest << VERSION << "\0" << @inputs << "\0" << @suite_digest << "\0" << subject.source << "\0"
102
+ digest << [VERSION, @inputs, @suite_digest, source_set, subject.source].join("\0") << "\0"
82
103
  # A hex digest has a fixed width, so it needs no separator after it.
83
104
  digest << Digest::SHA256.hexdigest(subject.file_source)
105
+ digest_tests(digest, tests)
106
+ hash_dependency_inputs(digest)
107
+ digest.hexdigest
108
+ end
109
+
110
+ def digest_tests(digest, tests)
84
111
  tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
112
+ dependencies(tests).each { |dep| digest << dep << "\0" << file_digest(dep) << "\0" }
85
113
  LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
86
- digest.hexdigest
114
+ end
115
+
116
+ # Files a change in which can flip the verdict (#197): every source
117
+ # file a covering test touched, from the run's coverage map, plus the
118
+ # loaded sources the map never observed (#344, #353).
119
+ def dependencies(tests)
120
+ (@coverage.files_for(tests) | @coverage.boot_files | unobserved_files).sort
121
+ end
122
+
123
+ # Loaded sources the map never observed (no line, method, or load
124
+ # evidence): coverage cannot say whether they matter, so a change in
125
+ # any of them must invalidate (#344).
126
+ def unobserved_files
127
+ @unobserved_files ||=
128
+ (@fallback_files | @coverage.loaded_files).select { |file| @coverage.empty_for?([file]) }
87
129
  end
88
130
 
89
131
  # RSpec ids name their file: "./spec/a_spec.rb[1:2]".
@@ -91,6 +133,21 @@ module Saboteur
91
133
  File.expand_path(test[/\A(.+?\.rb)(?:\[|:|\z)/, 1] || test, @root)
92
134
  end
93
135
 
136
+ def hash_dependency_inputs(digest)
137
+ dependency_files.each { |path| digest << path.delete_prefix("#{@root}/") << "\0" << file_digest(path) << "\0" }
138
+ end
139
+
140
+ # Manifests and gemspecs anywhere below the root, since a monorepo
141
+ # sub-gem carries its own dependency declarations (#561); keyed
142
+ # root-relative so same-named sub-gem files cannot collide. Vendored
143
+ # installs and dependencies' own checkouts hold gemspecs the project
144
+ # does not declare, so the scan mirrors RunFingerprint's exclusions.
145
+ def dependency_files
146
+ @dependency_files ||= Dir[File.join(@root, "**", "{#{MANIFESTS.join(',')},*.gemspec}")].reject do |path|
147
+ path.delete_prefix("#{@root}/").split("/").any? { |part| RunFingerprint::EXCLUDED_DIRECTORIES.include?(part) }
148
+ end
149
+ end
150
+
94
151
  def file_digest(path)
95
152
  @file_digests[path] ||= File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing"
96
153
  end