saboteur 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +215 -0
  3. data/README.md +44 -6
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +116 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/presets.rb +41 -1
  20. data/lib/saboteur/cli/preview_command.rb +26 -0
  21. data/lib/saboteur/cli/run_budget_options.rb +6 -1
  22. data/lib/saboteur/cli/run_command.rb +33 -0
  23. data/lib/saboteur/cli/run_output.rb +38 -0
  24. data/lib/saboteur/cli/run_reporting.rb +6 -5
  25. data/lib/saboteur/cli/run_validation.rb +54 -0
  26. data/lib/saboteur/cli/source_set.rb +0 -18
  27. data/lib/saboteur/cli.rb +29 -305
  28. data/lib/saboteur/config.rb +76 -7
  29. data/lib/saboteur/config_file.rb +12 -8
  30. data/lib/saboteur/constant_subject_finder.rb +3 -0
  31. data/lib/saboteur/custom_map.rb +24 -5
  32. data/lib/saboteur/deep_source.rb +49 -0
  33. data/lib/saboteur/dirty_tree_warning.rb +6 -4
  34. data/lib/saboteur/doctor.rb +10 -0
  35. data/lib/saboteur/engine/baseline_guard.rb +118 -15
  36. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  37. data/lib/saboteur/engine/boot.rb +11 -3
  38. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  39. data/lib/saboteur/engine/candidate_selection.rb +16 -5
  40. data/lib/saboteur/engine/candidates.rb +4 -1
  41. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  42. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  43. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  44. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  45. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  46. data/lib/saboteur/engine/coverage_store.rb +296 -21
  47. data/lib/saboteur/engine/diagnostics.rb +24 -7
  48. data/lib/saboteur/engine/distribution.rb +2 -2
  49. data/lib/saboteur/engine/gated_run.rb +4 -1
  50. data/lib/saboteur/engine/load_coverage.rb +72 -8
  51. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  52. data/lib/saboteur/engine/partial_merge.rb +40 -0
  53. data/lib/saboteur/engine/pipelines.rb +11 -3
  54. data/lib/saboteur/engine/reference_specs.rb +71 -0
  55. data/lib/saboteur/engine/result_cache.rb +68 -5
  56. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  57. data/lib/saboteur/engine/run_budget.rb +10 -7
  58. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  59. data/lib/saboteur/engine/sandbox_policy.rb +26 -5
  60. data/lib/saboteur/engine/scheduler.rb +33 -25
  61. data/lib/saboteur/engine/selection_announcement.rb +9 -3
  62. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  63. data/lib/saboteur/engine/shadow_run.rb +38 -0
  64. data/lib/saboteur/engine/subject_selection.rb +144 -64
  65. data/lib/saboteur/engine/suite_partition.rb +22 -0
  66. data/lib/saboteur/engine/test_guess.rb +186 -32
  67. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  68. data/lib/saboteur/engine.rb +44 -11
  69. data/lib/saboteur/environment_hints.rb +55 -0
  70. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  71. data/lib/saboteur/execution/external_runner.rb +123 -1
  72. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  73. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  74. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  75. data/lib/saboteur/execution/process_table.rb +7 -3
  76. data/lib/saboteur/execution/reevaluator.rb +7 -0
  77. data/lib/saboteur/execution/sandbox.rb +39 -20
  78. data/lib/saboteur/execution/seatbelt.rb +10 -3
  79. data/lib/saboteur/execution/supervisor.rb +42 -8
  80. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  81. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  82. data/lib/saboteur/execution/worker_pool.rb +48 -4
  83. data/lib/saboteur/frameworks/rails.rb +2 -5
  84. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  85. data/lib/saboteur/gates.rb +121 -28
  86. data/lib/saboteur/install.rb +6 -1
  87. data/lib/saboteur/integration/base.rb +19 -2
  88. data/lib/saboteur/integration/rspec.rb +145 -18
  89. data/lib/saboteur/line_hits.rb +81 -0
  90. data/lib/saboteur/method_calls.rb +91 -0
  91. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  92. data/lib/saboteur/mutation_source/builtin.rb +33 -21
  93. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  94. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  95. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  96. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  97. data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
  98. data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
  99. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  100. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  101. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  102. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  103. data/lib/saboteur/preview.rb +13 -0
  104. data/lib/saboteur/refinement_activations.rb +22 -10
  105. data/lib/saboteur/reporter/agent.rb +2 -2
  106. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  107. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  108. data/lib/saboteur/reporter/review_comments.rb +151 -0
  109. data/lib/saboteur/reporter/safe_out.rb +32 -0
  110. data/lib/saboteur/reporter/stryker.rb +8 -6
  111. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  112. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  113. data/lib/saboteur/reporter/terminal.rb +45 -5
  114. data/lib/saboteur/result.rb +14 -4
  115. data/lib/saboteur/selection/coverage_map.rb +150 -9
  116. data/lib/saboteur/selection/external_test_map.rb +4 -0
  117. data/lib/saboteur/selection/git_diff.rb +24 -2
  118. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  119. data/lib/saboteur/selection/recorder.rb +52 -22
  120. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  121. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  122. data/lib/saboteur/subject.rb +16 -0
  123. data/lib/saboteur/subject_finder.rb +91 -6
  124. data/lib/saboteur/text.rb +31 -0
  125. data/lib/saboteur/version.rb +1 -1
  126. data/lib/saboteur/working_tree.rb +7 -2
  127. data/lib/saboteur.rb +9 -0
  128. metadata +48 -16
@@ -11,13 +11,29 @@ module Saboteur
11
11
  # unchanged, otherwise built by running the whole suite once in a
12
12
  # supervised child and saved.
13
13
  class CoverageStore
14
- CACHE_INPUTS = %w[integration require rspec_args rails tests].freeze
14
+ include CoverageShadow
15
+ include CoveragePartial
16
+ include CoverageParts
17
+
18
+ CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
15
19
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
16
20
  LOG = "coverage-build.log"
21
+ # RSpec rewrites this on every run (example_status_persistence_file_path),
22
+ # so hashing it would make every second run look edited.
23
+ GENERATED_OUTPUT = "spec/examples.txt"
24
+ # A snapshot entry that is never a file, so a snapshot holding it is never
25
+ # reused partially. It stands for a test file outside the project root,
26
+ # whose own inputs the snapshot cannot see.
27
+ OUTSIDE_TESTS = "(test file outside the project root)"
28
+ # What a failed child's log shows in the error: its last lines, bounded.
29
+ TAIL_LINES = 60
30
+ TAIL_BYTES = 65_536
31
+ TAIL_LINE_WIDTH = 300
17
32
 
18
33
  def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
19
- timeout_cap: nil)
34
+ timeout_cap: nil, parts: 1)
20
35
  @config = config
36
+ @parts = parts
21
37
  @integration = integration
22
38
  @hooks = hooks
23
39
  @supervisor = supervisor
@@ -40,11 +56,18 @@ module Saboteur
40
56
 
41
57
  def fetch(files)
42
58
  path = cache_path(files)
59
+ preloaded = LoadCoverage.preloaded_uninstrumented(files)
43
60
  cached = cached_map(files)
44
- return refuse_empty_map(cached, files) if cached
61
+ return refuse_empty_map(cached.with_preloaded(preloaded), files, preloaded) if cached
45
62
 
46
- map = refuse_empty_map(build(files), files)
63
+ note = miss_note(files)
64
+ @diagnostics.inform(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
65
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
66
+ map = fresh_map(files, note, preloaded)
67
+ build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
47
68
  map.save(path)
69
+ shadow(files, map, note, build_seconds) if shadow?
70
+ note.save
48
71
  File.write(File.join(@config.path("cache_dir"), "preview.json"),
49
72
  JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
50
73
  map
@@ -52,6 +75,12 @@ module Saboteur
52
75
 
53
76
  private
54
77
 
78
+ # The partial map when it is on and safe, else the full build.
79
+ def fresh_map(files, note, preloaded)
80
+ (partial_map(files, note, preloaded) if partial_use?) ||
81
+ refuse_empty_map(build(files).with_preloaded(preloaded), files, preloaded)
82
+ end
83
+
55
84
  def write_map(writer, recorder, build_seconds)
56
85
  writer.write(recorder.result(build_seconds: build_seconds, loaded_files: loaded_project_files).to_json)
57
86
  writer.close
@@ -78,9 +107,63 @@ module Saboteur
78
107
  end
79
108
 
80
109
  def cache_key(files)
81
- inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
82
- all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
83
- Selection::CoverageMap.cache_key(files: all.uniq, extra: inputs)
110
+ Selection::CoverageMap.cache_key(files: key_files(files), extra: key_inputs)
111
+ end
112
+
113
+ def key_inputs
114
+ JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
115
+ end
116
+
117
+ def recorder_class
118
+ @config["coverage_recorder"] == "trace" ? Selection::TraceRecorder : Selection::Recorder
119
+ end
120
+
121
+ def key_files(files)
122
+ (files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles).uniq
123
+ end
124
+
125
+ def miss_note(files)
126
+ CoverageMissNote.new(cache_dir: @config.path("cache_dir"), root: @root, files: snapshot_files(files),
127
+ settings: key_inputs, map_key: cache_key(files), seed: @config["seed"],
128
+ clean: ->(path) { plain_line(Text.utf8(path.to_s)) })
129
+ end
130
+
131
+ # What the inputs snapshot hashes: the cache key's files plus every file
132
+ # under spec/ and under the suite's own test directories, fixtures and
133
+ # config included, except the generated spec/examples.txt. A partial
134
+ # rebuild (#46) must see a change to anything a test can read, not only
135
+ # to what the key covers. Symlinks and other special entries are listed
136
+ # but never read (see CoverageInputsSnapshot.file_digest). The cache dir
137
+ # is left out so the pair never hashes itself.
138
+ def snapshot_files(files)
139
+ cache = File.join(File.expand_path(@config.path("cache_dir")), "")
140
+ dirs, outside = test_dirs
141
+ tree = dirs.flat_map do |dir|
142
+ Dir.glob("**/*", File::FNM_DOTMATCH, base: File.join(@root, dir)).map { |name| File.join(@root, dir, name) }
143
+ end
144
+ generated = File.join(@root, GENERATED_OUTPUT)
145
+ tree = tree.reject do |file|
146
+ file == generated || real_directory?(file) || File.expand_path(file).start_with?(cache)
147
+ end
148
+ marker = outside ? [OUTSIDE_TESTS] : []
149
+ [key_files(files), tree, marker].flatten.sort
150
+ end
151
+
152
+ # spec/ always, plus the first directory of every configured test file
153
+ # (a suite under test/ or elsewhere keeps its fixtures beside it), and
154
+ # whether a test file lies outside the project root. Both sides are
155
+ # compared as real paths, so a symlinked root still matches.
156
+ def test_dirs
157
+ prefix = "#{Selection::CoverageMap.canonical(@root)}/"
158
+ inside, outside = @integration.test_files.map { |file| Selection::CoverageMap.canonical(file) }
159
+ .partition { |file| file.start_with?(prefix) }
160
+ [["spec"] + inside.map { |file| file.delete_prefix(prefix).split("/").first }, !outside.empty?]
161
+ end
162
+
163
+ def real_directory?(file)
164
+ File.lstat(file).directory?
165
+ rescue SystemCallError
166
+ true
84
167
  end
85
168
 
86
169
  def lockfiles
@@ -99,6 +182,9 @@ module Saboteur
99
182
  "Run through the saboteur executable and disable SimpleCov for the run."
100
183
  end
101
184
 
185
+ parts = usage_checked { suite_parts }
186
+ return build_parts(files, parts) if parts.size > 1
187
+
102
188
  scope = @config["tests"].empty? ? "the whole suite" : "the selected specs"
103
189
  @diagnostics.inform("building coverage map (runs #{scope} once)")
104
190
  # The suite child forked next inherits this heap; pack it first, to
@@ -113,13 +199,136 @@ module Saboteur
113
199
  map_from(result, admitted)
114
200
  end
115
201
 
116
- def map_from(result, admitted = @config["coverage_timeout"])
202
+ # The spec files dealt into @parts disjoint parts; one part (the
203
+ # default, or a suite with fewer files than parts) keeps the single
204
+ # child build (#741).
205
+ def suite_parts
206
+ return [] unless @parts > 1
207
+
208
+ Engine::SuitePartition.parts(@integration.test_files, @parts)
209
+ end
210
+
211
+ # The same suite pass as build, run as several children at once, each
212
+ # over its own spec files. Fails loudly unless the parts hold exactly
213
+ # the suite's examples before the run and the combined map holds
214
+ # exactly its tests after it, and refuses a part that recorded no test.
215
+ def build_parts(files, parts)
216
+ total, counts = usage_checked { check_parts_cover_suite(parts) }
217
+ saved = saved_parts(files, parts.size, counts)
218
+ todo = saved.each_index.select { |index| saved[index].nil? }
219
+ @diagnostics.inform(parts_note(parts.size, todo.size))
220
+ built = todo.empty? ? [] : run_parts(files, parts, todo)
221
+ todo.zip(built) { |index, map| saved[index] = map }
222
+ map = combine_saved(saved, total, files, parts.size)
223
+ forget_parts(files, parts.size)
224
+ map
225
+ end
226
+
227
+ # Parts that cannot be combined are dropped, not kept: a bad saved
228
+ # part would otherwise fail every later run the same way.
229
+ def combine_saved(maps, total, files, count)
230
+ combine_parts(maps, total)
231
+ rescue ArgumentError => e
232
+ forget_parts(files, count)
233
+ raise Error, "the saved coverage parts do not combine (#{e.message}); they were removed, run again"
234
+ end
235
+
236
+ def parts_note(count, missing)
237
+ return "building coverage map (reusing all #{count} saved parts)" if missing.zero?
238
+
239
+ reused = count - missing
240
+ suffix = reused.zero? ? "" : ", reusing #{reused} saved"
241
+ "building coverage map (runs the suite once, in #{count} parallel parts#{suffix})"
242
+ end
243
+
244
+ # Runs only the parts in todo, as several children at once.
245
+ def run_parts(files, parts, todo)
246
+ Execution::ForkReadiness.prepare_heap
247
+ heartbeat = @diagnostics.progress_heartbeat("building coverage map")
248
+ admitted = build_timeout
249
+ results = @supervisor.run_many(todo.size, timeout: admitted, on_tick: heartbeat.method(:tick),
250
+ label: "coverage child died during the map build") do |writer, slot|
251
+ index = todo[slot]
252
+ own_test_env(index)
253
+ record(writer, files, only: parts[index], log: part_log_path(index))
254
+ end
255
+ part_maps(results, admitted, parts.size, files, todo)
256
+ end
257
+
258
+ # Every part that failed is reported, each with its own output, so one
259
+ # run shows all the failures instead of the first. A budget stop still
260
+ # raises at once: it says nothing about the suite. A part that did
261
+ # finish is saved either way, so the work is not lost (#427).
262
+ def part_maps(results, admitted, count, files, todo)
263
+ failures = []
264
+ budget_stop = nil
265
+ maps = results.each_with_index.map do |result, slot|
266
+ index = todo[slot]
267
+ map_from(result, admitted, log: part_log_path(index)).tap { |map| save_part(map, files, index, count) }
268
+ rescue BudgetPhaseTimeout => e
269
+ budget_stop = e
270
+ nil
271
+ rescue Error => e
272
+ failures << "part #{index + 1} of #{count}: #{e.message}"
273
+ nil
274
+ end
275
+ raise budget_stop if budget_stop
276
+ raise Error, failures.join("\n") unless failures.empty?
277
+
278
+ maps.each_with_index { |map, slot| part_map(map, todo[slot], count) }
279
+ end
280
+
281
+ # A part child gets its own TEST_ENV_NUMBER, as a WorkerPool worker
282
+ # does, so suites that key databases, ports or tmp dirs on it do not
283
+ # collide between parts (part 0 keeps the empty value).
284
+ def own_test_env(index)
285
+ ENV["TEST_ENV_NUMBER"] = index.zero? ? "" : (index + 1).to_s
286
+ end
287
+
288
+ # A bad file list is the caller's mistake: a usage error, not a crash.
289
+ def usage_checked
290
+ yield
291
+ rescue ArgumentError => e
292
+ raise ConfigError, e.message
293
+ end
294
+
295
+ def check_parts_cover_suite(parts)
296
+ total = @integration.example_count
297
+ counts = parts.map { |part| @integration.example_count(files: part) }
298
+ return [total, counts] if counts.sum == total
299
+
300
+ raise Error, "the coverage parts hold #{counts.sum} examples but the suite has #{total}; " \
301
+ "refusing to build a map from a partition that does not cover the suite"
302
+ end
303
+
304
+ def part_map(map, index, total)
305
+ return map unless map.tests.empty?
306
+
307
+ raise Error, "coverage part #{index + 1} of #{total} recorded no tests; refusing to build a partial map"
308
+ end
309
+
310
+ def combine_parts(maps, total)
311
+ map = Selection::CoverageMap.combine(maps)
312
+ return map if map.tests.size == total
313
+
314
+ raise Error, "the combined coverage map holds #{map.tests.size} tests but the suite has #{total}; " \
315
+ "refusing to use a partial map"
316
+ end
317
+
318
+ def part_log_path(index)
319
+ File.join(@config.path("cache_dir"), "coverage-build-#{index + 1}.log")
320
+ end
321
+
322
+ def map_from(result, admitted = @config["coverage_timeout"], log: log_path)
117
323
  raise_budget_stop(admitted) if budget_stopped?(result, admitted)
118
324
  raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
119
- raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
325
+ if result.payload.empty?
326
+ raise Error,
327
+ "coverage map build failed: #{died(result.exit_status, log)}#{output_tail(log)}"
328
+ end
120
329
 
121
330
  data = JSON.parse(result.payload)
122
- raise Error, "coverage map build failed: #{data['error']}" if data["error"]
331
+ raise Error, "coverage map build failed: #{child_error(data['error'])}#{output_tail(log)}" if data["error"]
123
332
 
124
333
  with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
125
334
  end
@@ -157,30 +366,96 @@ module Saboteur
157
366
  # these sources, so the map passes with a warning. Applies to a cached
158
367
  # map too, so a map cached before this check cannot keep a run green,
159
368
  # and the guard runs before saving, so an empty map is never cached.
160
- def refuse_empty_map(map, files)
369
+ def refuse_empty_map(map, files, preloaded)
161
370
  return map if files.empty? || !map.empty_for?(files)
162
371
 
372
+ # A file loaded before the executable's Coverage.start can never be
373
+ # instrumented, so emptiness there is a setup defect, not untested
374
+ # code: name it apart from the generic no-coverage refusal.
375
+ preloaded_empty = preloaded & files.map { |file| Selection::CoverageMap.canonical(file) }
376
+ if preloaded_empty.any?
377
+ message = preloaded_message(preloaded_empty)
378
+ raise Error, message if @config["tests"].empty?
379
+
380
+ @diagnostics.warn(message)
381
+ return map
382
+ end
163
383
  if @config["tests"].empty?
164
384
  raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
165
385
  "build. No test in the suite may exercise these files, or the setup hid the app " \
166
386
  "from Coverage#{bootsnap_note}. Run `saboteur doctor`."
167
387
  end
168
- @diagnostics.warn("Coverage recorded nothing in #{files.size} included files; " \
388
+ @diagnostics.warn("name the specs that exercise them in --test, or drop --test to run the whole suite\n " \
389
+ "Coverage recorded nothing in #{files.size} included files; " \
169
390
  "the selected specs may not exercise them.")
170
391
  map
171
392
  end
172
393
 
394
+ def preloaded_message(files)
395
+ names = Text.name_list(files.map { |file| file.delete_prefix("#{@root}/") })
396
+ "#{names} loaded before saboteur started coverage (a require ran before the " \
397
+ "executable's Coverage.start - commonly the gemspec requiring the library while bundler/setup " \
398
+ "evaluates it, or RUBYOPT -r), so no test can ever cover #{files.size > 1 ? 'them' : 'it'}. " \
399
+ "Let the test suite load these " \
400
+ "files; a gemspec should require only its version file."
401
+ end
402
+
173
403
  # Bootsnap is named only when it is loaded: blaming a library the app
174
404
  # does not use sends the reader to the wrong place.
175
405
  def bootsnap_note
176
406
  " (Bootsnap's compile cache is the default Rails case)" if defined?(::Bootsnap)
177
407
  end
178
408
 
179
- def died(status)
409
+ def died(status, log)
180
410
  how = if status&.signaled? then "was killed by signal #{Signal.signame(status.termsig)}"
181
411
  else "exited with status #{status&.exitstatus}"
182
412
  end
183
- "the suite #{how} before reporting; its output is in #{log_path.delete_prefix("#{@root}/")}"
413
+ "the suite #{how} before reporting; its output is in #{log.delete_prefix("#{@root}/")}"
414
+ end
415
+
416
+ # The end of a failed child's log, where rspec prints its failures, so a
417
+ # CI run shows why without the cache directory. The suite's own output
418
+ # and nothing else. Every line is cleaned by plain_line and starts with
419
+ # "| ", so it is never the start of a workflow command.
420
+ def output_tail(log)
421
+ size = File.size(log)
422
+ cut = size > TAIL_BYTES
423
+ text = File.open(log) do |file|
424
+ file.seek(size - TAIL_BYTES) if cut
425
+ file.read
426
+ end
427
+ lines = Text.utf8(text).lines
428
+ lines.shift if cut
429
+ lines = lines.last(TAIL_LINES).map { |line| tail_line(line) }
430
+ return "" if lines.empty?
431
+
432
+ "\n the last #{lines.size} lines of #{log.delete_prefix("#{@root}/")}:\n#{lines.join("\n")}"
433
+ rescue SystemCallError
434
+ ""
435
+ end
436
+
437
+ def tail_line(line)
438
+ "| #{plain_line(line)}"
439
+ end
440
+
441
+ # One line of a child's text, safe to print: colors stripped, control
442
+ # characters escaped, cut to a readable width, and the two things
443
+ # GitHub Actions reads as a workflow command broken where they stand: a
444
+ # "::" that starts the line after any leading whitespace (the runner
445
+ # trims all Unicode spaces first) and a "##[" anywhere in it. Other
446
+ # colons and brackets, as in "Saboteur::Error", are left alone.
447
+ def plain_line(line)
448
+ text = Text.display_safe(line.chomp.gsub(/\e\[[0-9;]*[A-Za-z]/, ""))
449
+ text.size > TAIL_LINE_WIDTH ? "#{text[0, TAIL_LINE_WIDTH]}..." : text
450
+ end
451
+
452
+ # The error a child wrote into its report. It is the child's text, which
453
+ # can carry anything the suite printed or raised, so it gets the log's
454
+ # treatment: the first line follows the message, every later line starts
455
+ # with "| ", and plain_line has cleaned them all.
456
+ def child_error(error)
457
+ first, *rest = Text.utf8(error.to_s).lines
458
+ [plain_line(first.to_s), *rest.first(TAIL_LINES).map { |line| tail_line(line) }].join("\n")
184
459
  end
185
460
 
186
461
  def log_path
@@ -189,9 +464,9 @@ module Saboteur
189
464
 
190
465
  # Runs in the supervised child. The suite's output goes to a log, so a
191
466
  # child that dies without reporting can still be diagnosed.
192
- def record(writer, files)
193
- FileUtils.mkdir_p(File.dirname(log_path))
194
- $stdout.reopen(log_path, "w")
467
+ def record(writer, files, only: nil, log: log_path, seed: @config["seed"])
468
+ FileUtils.mkdir_p(File.dirname(log))
469
+ $stdout.reopen(log, "w")
195
470
  $stdout.sync = true
196
471
  $stderr.reopen($stdout)
197
472
  @hooks.run(:after_fork)
@@ -199,13 +474,13 @@ module Saboteur
199
474
  # The suite's own files are recorded too: a spec can read a
200
475
  # constant directly, and without its line hits no test maps to the
201
476
  # read (#319).
202
- recorder = Selection::Recorder.new(files + @integration.test_files)
477
+ recorder = recorder_class.new(files + @integration.test_files)
203
478
  recorder.start
204
- Kernel.srand(@config["seed"])
205
- passed = @integration.run_all(recorder)
479
+ Kernel.srand(seed)
480
+ passed = only ? @integration.run_all(recorder, files: only, seed: seed) : @integration.run_all(recorder)
206
481
  if passed == false
207
482
  raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
208
- "with seed #{@config['seed']}."
483
+ "with seed #{seed}."
209
484
  end
210
485
  write_map(writer, recorder, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
211
486
  rescue Exception => e # rubocop:disable Lint/RescueException
@@ -73,8 +73,8 @@ module Saboteur
73
73
  Omission = Data.define(:subject, :count, :limit)
74
74
  DiscoveryFailure = Data.define(:file, :reason)
75
75
 
76
- attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
77
- :hazard_suppressions, :inline_suppressions
76
+ attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
77
+ :hazard_suppressions, :inline_suppressions, :verbose
78
78
 
79
79
  def initialize(err, verbose: false)
80
80
  @err = err
@@ -82,6 +82,7 @@ module Saboteur
82
82
  @warnings = []
83
83
  @invalid_mutations = 0
84
84
  @generation_events = []
85
+ @live_announced = []
85
86
  @omissions = []
86
87
  @discovery_failures = []
87
88
  @hazard_suppressions = 0
@@ -90,8 +91,19 @@ module Saboteur
90
91
  @erb_template_note_emitted = false
91
92
  end
92
93
 
94
+ # Every warning passes through here once: repository-controlled text
95
+ # (file names, patterns, thread names, error text) cannot start a CI
96
+ # workflow command on a line of its own, whichever site built it
97
+ # (#834). Already escaped text is unchanged.
93
98
  def warn(message)
94
- @warnings << message
99
+ @warnings << Text.display_safe_lines(message)
100
+ end
101
+
102
+ # Warnings built elsewhere (the ignore file's, the custom map's, a
103
+ # child's): each goes through #warn, so none skips the escaping.
104
+ def add_warnings(messages)
105
+ messages.each { |message| warn(message) }
106
+ nil
95
107
  end
96
108
 
97
109
  # ERB generator templates (Rails generators carry `.rb` files with ERB
@@ -148,7 +160,7 @@ module Saboteur
148
160
  end
149
161
 
150
162
  def inform(message)
151
- @err.puts("saboteur: #{message}")
163
+ @err.puts("saboteur: #{Text.display_safe_lines(message)}")
152
164
  end
153
165
 
154
166
  # --verbose: one line per finished subject with its result marks, so a
@@ -162,10 +174,15 @@ module Saboteur
162
174
  inform("#{subject.id} #{marks}")
163
175
  end
164
176
 
165
- # A message that is both a report warning and a stderr note.
177
+ # A message that is both a report warning and a stderr note. The
178
+ # terminal does not reprint it at the end of the run (#605), but it
179
+ # stays in report.warnings: JSON, HTML and agent consumers discard
180
+ # stderr and must still see it.
166
181
  def announce(message)
167
- warn(message)
168
- inform(message)
182
+ safe = Text.display_safe_lines(message)
183
+ warn(safe)
184
+ inform(safe)
185
+ @live_announced << safe
169
186
  end
170
187
 
171
188
  # A heartbeat for a long phase: the supervisor's wait loop calls tick
@@ -192,11 +192,11 @@ module Saboteur
192
192
 
193
193
  def rebuild(row, subject, list)
194
194
  if row["error"]
195
- @diagnostics.warn("#{subject.id}: #{row['error']}")
195
+ @diagnostics.warn("#{subject.id}: its mutants are reported as errors\n #{row['error']}")
196
196
  return failed(row["error"], subject, list)
197
197
  end
198
198
 
199
- @diagnostics.warnings.concat(row["warnings"])
199
+ @diagnostics.add_warnings(row["warnings"])
200
200
  decode(row, subject, list).each { |result| @cache&.record(result) unless result.reused? }
201
201
  end
202
202
 
@@ -18,6 +18,7 @@ module Saboteur
18
18
  def report_unmapped(guess)
19
19
  @diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
20
20
  inform_skipped_specs(guess)
21
+ @diagnostics.announce(guess.referenced_message) if guess.referenced.any?
21
22
  stop_or_warn_unmapped(guess)
22
23
  end
23
24
 
@@ -26,7 +27,9 @@ module Saboteur
26
27
  # a gated run keep the hard stop.
27
28
  def stop_or_warn_unmapped(guess)
28
29
  return if guess.unmatched.empty?
29
- raise ConfigError, guess.unmatched_message if !config["since"] || guess.specs.empty? || gated?
30
+ if !config["since"] || guess.specs.empty? || gated?
31
+ raise ConfigError, guess.unmatched_message(verbose: @diagnostics.verbose)
32
+ end
30
33
 
31
34
  guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
32
35
  end
@@ -19,26 +19,90 @@ module Saboteur
19
19
  return [{}, {}] unless Coverage.state == :suspended
20
20
 
21
21
  wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
22
+ calls = boot_calls(wanted)
22
23
  CoverageReads.result(stop: false, clear: false)
23
24
  .each_with_object([{}, {}]) do |(path, data), (lines_h, methods_h)|
24
25
  file = Selection::CoverageMap.canonical(path)
25
26
  next unless wanted.include?(file)
26
27
 
27
- lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
28
- lines -= uncalled_shared_lines(data)
28
+ called = calls.fetch(file, Set.new)
29
+ lines = ran_lines(file, data, called)
29
30
  lines_h[file] = lines unless lines.empty?
30
- defs = data.fetch(:methods, {}).filter_map { |(_owner, _name, first, *), count| first if count.positive? }
31
- methods_h[file] = defs.uniq.sort unless defs.empty?
31
+ defs = called.map(&:first).uniq.sort
32
+ methods_h[file] = defs unless defs.empty?
32
33
  end
33
34
  end
34
35
 
36
+ def self.ran_lines(file, data, called)
37
+ lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
38
+ lines - uncalled_shared_lines(file, called)
39
+ end
40
+
41
+ # The def calls the executable's tracer saw while the app loaded,
42
+ # file => Set of [line, name]. Coverage's methods report is off (its
43
+ # per-read cost is what made the coverage map slow, #655).
44
+ def self.boot_calls(wanted)
45
+ tracer = MethodCalls.booted
46
+ return {} unless tracer
47
+
48
+ tracer.disable
49
+ tracer.drain.each_with_object({}) do |(path, hits), by_file|
50
+ file = Selection::CoverageMap.canonical(path)
51
+ (by_file[file] ||= Set.new).merge(hits) if wanted.include?(file)
52
+ end
53
+ end
54
+
55
+ # Include-listed files that are already loaded yet carry no coverage
56
+ # instrumentation: something required them before the executable's
57
+ # Coverage.start (a gemspec requiring the library while bundler/setup
58
+ # evaluates it, or RUBYOPT -r), so no test can ever be attributed to
59
+ # them. Read in the parent, where the snapshot still reflects the boot.
60
+ def self.preloaded_uninstrumented(files, loaded: $LOADED_FEATURES)
61
+ return [] unless Coverage.state == :suspended
62
+
63
+ instrumented = CoverageReads.result(stop: false, clear: false)
64
+ .each_key
65
+ .to_set { |path| Selection::CoverageMap.canonical(path) }
66
+ loaded_set = loaded.to_set { |path| Selection::CoverageMap.canonical(path) }
67
+ files.map { |file| Selection::CoverageMap.canonical(file) }
68
+ .uniq
69
+ .reject { |file| instrumented.include?(file) }
70
+ .select { |file| loaded_set.include?(file) }
71
+ end
72
+
35
73
  # A one-line or endless definition executes its def line even when its
36
74
  # body is never called. If any sibling on that physical line is uncalled,
37
75
  # discard its ambiguous line evidence rather than crediting every body.
38
- def self.uncalled_shared_lines(data)
39
- shared = data.fetch(:methods, {}).select { |(_owner, _name, first, _column, last, *), _| first == last }
40
- calls = shared.group_by { |(_owner, _name, first, *), _count| first }
41
- calls.filter_map { |line, entries| line if entries.any? { |_key, count| count.zero? } }
76
+ def self.uncalled_shared_lines(file, called)
77
+ defs = one_line_defs(file)
78
+ defs.group_by(&:first).filter_map do |line, entries|
79
+ line if entries.any? { |entry| !called.include?(entry) }
80
+ end
81
+ end
82
+
83
+ DEFINERS = %i[define_method define_singleton_method].freeze
84
+
85
+ # [line, name] for every def, or define_method with a block, that starts
86
+ # and ends on one physical line. Coverage's methods report lists both.
87
+ def self.one_line_defs(file)
88
+ found = []
89
+ walk = lambda do |node|
90
+ name = definition_name(node)
91
+ found << [node.location.start_line, name] if name && node.location.start_line == node.location.end_line
92
+ node.compact_child_nodes.each(&walk)
93
+ end
94
+ walk.call(Prism.parse_file(file).value)
95
+ found
96
+ rescue SystemCallError
97
+ []
98
+ end
99
+
100
+ def self.definition_name(node)
101
+ return node.name if node.is_a?(Prism::DefNode)
102
+ return unless node.is_a?(Prism::CallNode) && DEFINERS.include?(node.name) && node.block.is_a?(Prism::BlockNode)
103
+
104
+ first = Array(node.arguments&.arguments).first
105
+ first.unescaped.to_sym if first.is_a?(Prism::SymbolNode)
42
106
  end
43
107
  end
44
108
  end