saboteur 0.7.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +772 -0
  3. data/README.md +221 -1072
  4. data/lib/saboteur/attribute_finder.rb +113 -0
  5. data/lib/saboteur/cli/exit_status.rb +32 -1
  6. data/lib/saboteur/cli/help_options.rb +2 -1
  7. data/lib/saboteur/cli/history_command.rb +135 -0
  8. data/lib/saboteur/cli/presets.rb +47 -3
  9. data/lib/saboteur/cli/run_budget_options.rb +3 -1
  10. data/lib/saboteur/cli/selection_options.rb +31 -0
  11. data/lib/saboteur/cli/shard_options.rb +17 -0
  12. data/lib/saboteur/cli/source_set.rb +35 -0
  13. data/lib/saboteur/cli/survivors_command.rb +157 -0
  14. data/lib/saboteur/cli.rb +74 -55
  15. data/lib/saboteur/config.rb +102 -6
  16. data/lib/saboteur/config_file.rb +19 -7
  17. data/lib/saboteur/constant_reads.rb +182 -0
  18. data/lib/saboteur/constant_subject.rb +70 -0
  19. data/lib/saboteur/constant_subject_finder.rb +281 -0
  20. data/lib/saboteur/coverage_reads.rb +29 -0
  21. data/lib/saboteur/custom_map.rb +12 -1
  22. data/lib/saboteur/dirty_tree_warning.rb +130 -0
  23. data/lib/saboteur/doctor.rb +13 -0
  24. data/lib/saboteur/engine/baseline_guard.rb +135 -21
  25. data/lib/saboteur/engine/boot.rb +11 -3
  26. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  27. data/lib/saboteur/engine/candidate_selection.rb +20 -0
  28. data/lib/saboteur/engine/candidates.rb +43 -3
  29. data/lib/saboteur/engine/coverage_store.rb +58 -14
  30. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  31. data/lib/saboteur/engine/diagnostics.rb +19 -1
  32. data/lib/saboteur/engine/distribution.rb +10 -0
  33. data/lib/saboteur/engine/early_stop_recording.rb +3 -2
  34. data/lib/saboteur/engine/gated_run.rb +37 -0
  35. data/lib/saboteur/engine/load_coverage.rb +63 -0
  36. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  37. data/lib/saboteur/engine/pipelines.rb +115 -0
  38. data/lib/saboteur/engine/result_cache.rb +65 -8
  39. data/lib/saboteur/engine/run_budget.rb +32 -2
  40. data/lib/saboteur/engine/sandbox_policy.rb +59 -3
  41. data/lib/saboteur/engine/scheduler.rb +131 -17
  42. data/lib/saboteur/engine/selection_announcement.rb +59 -0
  43. data/lib/saboteur/engine/shard.rb +69 -0
  44. data/lib/saboteur/engine/subject_selection.rb +380 -21
  45. data/lib/saboteur/engine/test_guess.rb +407 -27
  46. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  47. data/lib/saboteur/engine.rb +103 -129
  48. data/lib/saboteur/execution/external_runner.rb +489 -0
  49. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  50. data/lib/saboteur/execution/marker_sweep.rb +80 -0
  51. data/lib/saboteur/execution/peak_memory.rb +28 -0
  52. data/lib/saboteur/execution/process_table.rb +19 -3
  53. data/lib/saboteur/execution/process_tree.rb +65 -5
  54. data/lib/saboteur/execution/reevaluator.rb +75 -12
  55. data/lib/saboteur/execution/runner.rb +46 -7
  56. data/lib/saboteur/execution/sandbox.rb +25 -6
  57. data/lib/saboteur/execution/seatbelt.rb +233 -0
  58. data/lib/saboteur/execution/supervisor.rb +29 -11
  59. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  60. data/lib/saboteur/execution/worker_pool.rb +23 -1
  61. data/lib/saboteur/frameworks/sqlite_memory_database.rb +79 -0
  62. data/lib/saboteur/gates.rb +43 -2
  63. data/lib/saboteur/help.rb +60 -9
  64. data/lib/saboteur/ignore_list.rb +29 -6
  65. data/lib/saboteur/integration/rspec.rb +223 -5
  66. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  67. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  68. data/lib/saboteur/mutation_source/builtin.rb +28 -5
  69. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  70. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  71. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  72. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  73. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  74. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  75. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  76. data/lib/saboteur/mutation_source/hazard_guard.rb +189 -0
  77. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  78. data/lib/saboteur/mutation_source/method_swap.rb +148 -0
  79. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  80. data/lib/saboteur/mutation_source/operator_examples.rb +4 -1
  81. data/lib/saboteur/mutation_source/operator_registry.rb +5 -2
  82. data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
  83. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  84. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  86. data/lib/saboteur/preview.rb +125 -12
  87. data/lib/saboteur/refinement_activations.rb +60 -0
  88. data/lib/saboteur/reporter/agent.rb +9 -1
  89. data/lib/saboteur/reporter/agent_commands.rb +10 -3
  90. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  91. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  92. data/lib/saboteur/reporter/findings.rb +5 -2
  93. data/lib/saboteur/reporter/history_policy.rb +11 -0
  94. data/lib/saboteur/reporter/history_reader.rb +47 -0
  95. data/lib/saboteur/reporter/html.rb +26 -4
  96. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  97. data/lib/saboteur/reporter/stryker.rb +27 -3
  98. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  99. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  100. data/lib/saboteur/reporter/terminal.rb +56 -8
  101. data/lib/saboteur/result.rb +19 -3
  102. data/lib/saboteur/selection/coverage_map.rb +120 -10
  103. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  104. data/lib/saboteur/selection/external_test_map.rb +27 -0
  105. data/lib/saboteur/selection/git_diff.rb +93 -11
  106. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  107. data/lib/saboteur/selection/recorder.rb +22 -4
  108. data/lib/saboteur/selection/source_scope.rb +22 -0
  109. data/lib/saboteur/subject.rb +42 -7
  110. data/lib/saboteur/subject_finder.rb +173 -29
  111. data/lib/saboteur/version.rb +1 -1
  112. data/lib/saboteur/working_tree.rb +38 -15
  113. data/lib/saboteur.rb +14 -0
  114. metadata +39 -16
@@ -39,12 +39,24 @@ module Saboteur
39
39
  rejectedKills: report.rejected_kills,
40
40
  warnings: report.warnings,
41
41
  framework: { name: "saboteur", version: VERSION },
42
- saboteur: { findings: Findings.build(report), scores: scores(report),
43
- inlineSuppressions: inline_audit(report) }
44
- .merge(operatorPolicy: report.operator_policy).merge(exit_cause ? { exitCause: exit_cause } : {})
42
+ saboteur: saboteur_enrichment(report, exit_cause)
45
43
  }.merge(report.seed ? { config: { seed: report.seed } } : {})
46
44
  end
47
45
 
46
+ # saboteur-namespaced keys beyond Stryker's schema: findings and
47
+ # scores, then the optional policy, shard, exit-cause and memory keys.
48
+ def self.saboteur_enrichment(report, exit_cause)
49
+ { findings: Findings.build(report), scores: scores(report),
50
+ inlineSuppressions: inline_audit(report) }
51
+ .merge(operatorPolicy: report.operator_policy).merge(shard(report))
52
+ .merge(exit_cause ? { exitCause: exit_cause } : {})
53
+ .merge(report.memory ? { memory: report.memory } : {})
54
+ end
55
+
56
+ def self.shard(report)
57
+ report.shard ? { shard: report.shard.to_h } : {}
58
+ end
59
+
48
60
  def self.inline_audit(report)
49
61
  report.inline_suppressions.map { |entry| entry.merge(file: relative(entry[:file], report.root)) }
50
62
  end
@@ -107,7 +119,14 @@ module Saboteur
107
119
 
108
120
  def self.optional_fields(result, test_ids)
109
121
  {
122
+ # A rejected kill is a verdict, not a tool error: the tests failed
123
+ # but also fail with the method unchanged. Flag it per mutant;
124
+ # the top-level rejectedKills stays a bare count (#375).
125
+ rejectedKill: (true if result.rejected_kill?),
110
126
  statusReason: status_reason(result),
127
+ # Why the survivor is worth a look, as a label only: scores and
128
+ # the exit code never change (#192).
129
+ classification: SurvivorClassification.label(result),
111
130
  killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
112
131
  coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
113
132
  duration: result.runtime && (result.runtime * 1000).round
@@ -129,6 +148,11 @@ module Saboteur
129
148
  end
130
149
 
131
150
  def self.status_reason(result)
151
+ return "todo: #{result.triage_reason}" if result.triage_state == "todo"
152
+ # The "ignored: " prefix is a public contract consumers classify by
153
+ # (#304 CI): accepted_equivalent stays byte-identical to a plain
154
+ # ignore, and only wont_fix adds a suffix after the reason.
155
+ return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
132
156
  return "ignored: #{result.ignored}" if result.ignored
133
157
  return "resource limit: #{result.reason}" if result.status == :resource_limit
134
158
  return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
@@ -0,0 +1,101 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "../engine/bytecode_equivalence"
5
+ require_relative "../execution/external_runner"
6
+ require_relative "../mutation_source/builtin"
7
+
8
+ module Saboteur
9
+ module Reporter
10
+ # Why a survivor is worth a look, as a LABEL ONLY (#192): the label
11
+ # lands in reports so a reader can sort real gaps from noise. Every
12
+ # label is a HEURISTIC, not a diagnosis - inspect the survivor before
13
+ # acting on it, and never add a test or ignore a mutant from the label
14
+ # alone. Labels never change scores or the exit code.
15
+ #
16
+ # One label per survivor, first match wins:
17
+ # likely_equivalent - the mutant compiles to the same instructions as
18
+ # the original (the #151/#169 bytecode filter): strong evidence of
19
+ # a no-op mutant, worth checking against the source. First: a
20
+ # no-op mutant's survival says nothing about the suite.
21
+ # arid - every line an edit touches matches an arid pattern
22
+ # (logging and friends, #111/#187). The builtin hint list decides;
23
+ # a configured arid policy does not change the label.
24
+ # missing_test - a test covered the line and passed, which MAY mean
25
+ # the suite reaches the code without pinning this behaviour. The
26
+ # covering list is only the PLANNED coverage: whole-body (coarse)
27
+ # mutants carry method-level guesses, so they never get this
28
+ # label, and neither does anything without real coverage.
29
+ # tool_limit - reserved for survivors saboteur could not observe
30
+ # faithfully (re-eval refusal, external runner without coverage). No
31
+ # result records that signal yet, so the label is never emitted.
32
+ #
33
+ # A survivor with no matching signal stays unlabeled.
34
+ module SurvivorClassification
35
+ LABELS = %w[likely_equivalent arid missing_test tool_limit].freeze
36
+
37
+ class << self
38
+ def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
39
+ return nil unless result.status == :survived
40
+ return "likely_equivalent" if equivalent?(result)
41
+ return "arid" if arid?(result, arid)
42
+ return "missing_test" if possible_test_gap?(result)
43
+
44
+ nil
45
+ end
46
+
47
+ private
48
+
49
+ # The mutant's own def text, compiled side by side with the
50
+ # original: equal instruction sequences mean behaviour cannot
51
+ # change. Edits index the whole file, so they shift by the def's
52
+ # start byte before splicing into the def text.
53
+ def equivalent?(result)
54
+ subject = result.subject
55
+ original = subject.source
56
+ mutated = result.mutation.edits.sort_by(&:start_byte).reverse.inject(original) do |text, edit|
57
+ from = edit.start_byte - subject.start_byte
58
+ to = edit.end_byte - subject.start_byte
59
+ text.byteslice(0, from) + edit.replacement + text.byteslice(to..)
60
+ end
61
+ Engine::BytecodeEquivalence.new.equivalent?(original, mutated,
62
+ frozen_string_literal: subject.frozen_literals)
63
+ end
64
+
65
+ # covered_by is a PLANNED covering list, not a verdict about what
66
+ # the suite asserts: coarse (whole-body) mutants carry method-level
67
+ # guesses, and a --test-command run records only the pseudo-test id
68
+ # for every mutant, so neither reads as real coverage (#305 QA).
69
+ def possible_test_gap?(result)
70
+ real_coverage = result.covered_by - [Execution::ExternalRunner::PSEUDO_TEST]
71
+ !result.mutation.coarse && real_coverage.any?
72
+ end
73
+
74
+ # Arid only when EVERY line an edit touches reads as arid: an edit
75
+ # that starts on a logging line but removes a composite body is not
76
+ # arid. The operator never decides: removing a logging statement is
77
+ # arid whatever removed it.
78
+ def arid?(result, patterns)
79
+ result.mutation.edits.all? do |edit|
80
+ lines = edited_lines(result.subject, edit)
81
+ lines.any? && lines.all? do |line|
82
+ patterns.any? { |pattern| line.strip.match?(Regexp.new(pattern)) }
83
+ end
84
+ end
85
+ end
86
+
87
+ # Edit offsets are BYTES, so the line lookup runs on the binary
88
+ # view (index/rindex are byte-exact there) and only the extracted
89
+ # lines are decoded, scrubbed so a partial multibyte sequence at an
90
+ # edit boundary cannot raise on the pattern match.
91
+ def edited_lines(subject, edit)
92
+ bytes = subject.file_source.b
93
+ last = [edit.end_byte - 1, edit.start_byte].max
94
+ line_start = (bytes.byteslice(0, edit.start_byte).rindex("\n") || -1) + 1
95
+ line_end = bytes.index("\n", last) || bytes.bytesize
96
+ bytes.byteslice(line_start, line_end - line_start).force_encoding(Encoding::UTF_8).scrub.lines
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -47,14 +47,16 @@ module Saboteur
47
47
  end
48
48
  end
49
49
 
50
- # Mutants on one line share the printed evidence only when it is the
51
- # same for all of them: covering tests (in order), reuse and changed
52
- # lines can each differ per mutant, so each run of neighbours with the
53
- # same tuple is one group and shares evidence above its own mutations.
54
- # Evidence that repeats after another group is printed again, which
55
- # keeps the ranked order.
50
+ # Mutants on one line share the printed evidence when it names the
51
+ # same tests: the covering-test list is compared as a set (#502), so
52
+ # one line's mutants no longer reprint an identical block only because
53
+ # the run measured the killer order per mutant. Reuse and changed
54
+ # lines still group exactly. Each run of neighbours with the same key
55
+ # is one group and shares evidence above its own mutations; evidence
56
+ # that repeats after another group is printed again, which keeps the
57
+ # ranked order. The block prints in the first mutant's run order.
56
58
  def evidence_groups(report, on_line)
57
- on_line.chunk { |result| [result.covered_by, result.reused?, report.changed_lines_for(result)] }
59
+ on_line.chunk { |result| [result.covered_by.uniq.sort, result.reused?, report.changed_lines_for(result)] }
58
60
  .map(&:last)
59
61
  end
60
62
 
@@ -70,9 +72,13 @@ module Saboteur
70
72
  end
71
73
 
72
74
  def mutation(result)
75
+ change = if MutationText.removal?(result)
76
+ "removes #{MutationText.original(result)}"
77
+ else
78
+ "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))}"
79
+ end
73
80
  text = " #{@palette.paint(:red, '-')} #{result.mutation.operator} " \
74
- "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))} " \
75
- "#{result.digest}"
81
+ "#{change} #{result.digest}"
76
82
  text += " (ignored: #{result.ignored})" if result.ignored
77
83
  @out.puts text
78
84
  hint(result)
@@ -87,10 +87,20 @@ module Saboteur
87
87
  end
88
88
  end
89
89
 
90
+ # An early stop runs fewer mutants than the selection: say both
91
+ # numbers so "4 mutants" is not read as the whole plan (#582).
92
+ def mutant_counts(report)
93
+ planned = report.early_stop&.planned_mutants
94
+ return "#{report.results.size} mutants" unless planned && planned > report.results.size
95
+
96
+ "#{planned} mutants selected, #{report.results.size} attempted"
97
+ end
98
+
90
99
  def summary(report)
91
100
  seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
92
- @out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
101
+ @out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{mutant_counts(report)}, " \
93
102
  "wall #{format('%.2f', report.wall_time)} s#{seed}"
103
+ @out.puts report.shard.line if report.shard
94
104
  @out.puts status_line(report)
95
105
  details(report)
96
106
  @score_table.call(report)
@@ -103,10 +113,12 @@ module Saboteur
103
113
  survived = "survived #{c[:survived]}"
104
114
  ignored = report.survivors.size - report.unignored_survivors.size
105
115
  survived += " (#{ignored} ignored)" if ignored.positive?
116
+ rejected = report.rejected_kills
117
+ rejected_note = " (#{rejected} rejected kill#{'s' if rejected > 1})" if rejected.positive?
106
118
  line = "#{@palette.paint(:green,
107
119
  "killed #{c[:killed]}")} #{@palette.paint(:red,
108
120
  survived)} timeout #{c[:timeout]} " \
109
- "resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
121
+ "resource limit #{c[:resource_limit]} error #{c[:error]}#{rejected_note} skipped #{c[:skipped]}"
110
122
  line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
111
123
  line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
112
124
  line += " unverified #{c[:unverified]}" if c[:unverified].positive?
@@ -122,11 +134,19 @@ module Saboteur
122
134
 
123
135
  def details(report)
124
136
  breakdown = report.score_breakdown
125
- @out.puts "score #{percent(breakdown.value)}" \
126
- "#{if @verbose
127
- "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
128
- 'errors, resource limits and skips excluded'
129
- end}"
137
+ # The share covers scored mutants only, so an error-dominated run
138
+ # must not print it as a bare score: "score 100.0 %" above "the run
139
+ # tested almost nothing" reads as a contradiction (#557).
140
+ if report.majority_errored?
141
+ @out.puts "score n/a (#{breakdown.tested} of #{report.executed.size} mutants scored, " \
142
+ "#{percent(breakdown.value)} of scored)"
143
+ else
144
+ @out.puts "score #{percent(breakdown.value)}" \
145
+ "#{if @verbose
146
+ "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
147
+ 'errors, resource limits and skips excluded'
148
+ end}"
149
+ end
130
150
  report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
131
151
  reused = report.results.count(&:reused?)
132
152
  @out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
@@ -233,6 +253,7 @@ module Saboteur
233
253
  label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
234
254
  @out.puts @palette.paint(:yellow, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
235
255
  "covers only completed work")
256
+ budget_shortfall(stop, report.results.size)
236
257
  list(stop.unreached) { |id| " not run: #{id}" }
237
258
  list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
238
259
  list(stop.unverified) { |id| " unverified: #{id}" }
@@ -243,6 +264,23 @@ module Saboteur
243
264
  @out.puts " ... #{entries.size - MAX_ERRORS} more" if entries.size > MAX_ERRORS
244
265
  end
245
266
 
267
+ def budget_shortfall(stop, completed)
268
+ return unless stop.reason == "max_duration"
269
+
270
+ # Exit 4 alone does not say whether anything was judged: headline
271
+ # it when the baselines and coverage map ate the whole budget (#637).
272
+ if completed.zero?
273
+ @out.puts @palette.paint(:yellow, "no mutants ran before the --max-duration budget was spent; " \
274
+ "there is no score to judge")
275
+ end
276
+ line = if stop.planned_mutants
277
+ "partial: #{stop.planned_mutants - completed} of #{stop.planned_mutants} mutants not run"
278
+ else
279
+ "partial: mutant count unknown; the budget was spent before planning finished"
280
+ end
281
+ @out.puts @palette.paint(:yellow, line)
282
+ end
283
+
246
284
  def early_stop_summary(stop)
247
285
  parts = []
248
286
  if stop.unreached.any?
@@ -264,10 +302,20 @@ module Saboteur
264
302
  end
265
303
 
266
304
  def errors(report)
267
- errored = report.results.select { |result| result.status == :error }
305
+ # Rejected kills are verdicts, not tool errors: the tests failed,
306
+ # but they also fail with the method unchanged (#375).
307
+ rejected, errored = report.results.select { |result| result.status == :error }.partition(&:rejected_kill?)
308
+ unless rejected.empty?
309
+ @out.puts "\nRejected kills (tests also fail with the method unchanged; not counted as killed):"
310
+ print_error_lines(rejected)
311
+ end
268
312
  return if errored.empty?
269
313
 
270
314
  @out.puts "\nErrors (not counted as killed):"
315
+ print_error_lines(errored)
316
+ end
317
+
318
+ def print_error_lines(errored)
271
319
  errored.first(MAX_ERRORS).each do |result|
272
320
  lines = result.reason.to_s.lines.map(&:strip)
273
321
  @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: #{lines.first}"
@@ -23,6 +23,11 @@ module Saboteur
23
23
  attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
24
24
  :kill_cause, :executed
25
25
  attr_accessor :ignored
26
+ # The ignore list's triage for this survivor (#155): a state from
27
+ # IgnoreList::STATES and its reason. accepted_equivalent and wont_fix
28
+ # suppress the exit failure (the survivor still counts in the score); a
29
+ # todo survivor keeps counting as an unignored survivor.
30
+ attr_accessor :triage_state, :triage_reason
26
31
 
27
32
  def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
28
33
  covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
@@ -41,6 +46,9 @@ module Saboteur
41
46
  @rejected_kill = rejected_kill
42
47
  @reused = reused
43
48
  @ignored = nil
49
+ # triage_state/triage_reason stay unset until the ignore list writes
50
+ # them: the accessors read as nil before that, and there is no
51
+ # statement for a mutation to remove (#304 dogfood).
44
52
  end
45
53
 
46
54
  # The same mutant, marked unverified: the budget ran out before its
@@ -117,7 +125,11 @@ module Saboteur
117
125
  # subjects that started but stopped mid-way as {subject:, omitted:}
118
126
  # entries; unverified lists the subjects whose kills the budget left
119
127
  # unchecked. The verdict covers only completed, verified work.
120
- EarlyStop = Data.define(:reason, :unreached, :truncated, :unverified)
128
+ EarlyStop = Data.define(:reason, :unreached, :truncated, :unverified, :planned_mutants) do
129
+ def initialize(reason:, unreached:, truncated:, unverified:, planned_mutants: nil)
130
+ super
131
+ end
132
+ end
121
133
 
122
134
  # Everything one run produced.
123
135
  class Report
@@ -127,14 +139,14 @@ module Saboteur
127
139
 
128
140
  attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
129
141
  :generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions,
130
- :operator_policy, :discovery_failures
142
+ :operator_policy, :discovery_failures, :memory, :shard
131
143
 
132
144
  # REVIEW: diff mode; the terminal then shows one survivor per line.
133
145
  # seed: the random seed every test run started from (nil when unknown).
134
146
  def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
135
147
  seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
136
148
  run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [],
137
- operator_policy: nil, discovery_failures: [])
149
+ operator_policy: nil, discovery_failures: [], memory: nil, shard: nil)
138
150
  @results = results
139
151
  @subjects = subjects
140
152
  @wall_time = wall_time
@@ -154,6 +166,10 @@ module Saboteur
154
166
  @inline_suppressions = inline_suppressions.dup.freeze
155
167
  @operator_policy = operator_policy
156
168
  @discovery_failures = discovery_failures.dup.freeze
169
+ # Per-worker and supervisor peak RSS in MB, when the platform measured
170
+ # it (#202); nil elsewhere, so JSON reports omit the section.
171
+ @memory = memory.dup.freeze
172
+ @shard = shard
157
173
  end
158
174
 
159
175
  # A --matrix run: every covering test ran per mutant.
@@ -17,28 +17,59 @@ module Saboteur
17
17
  # to every test in the file of the next test that starts. Each test's
18
18
  # duration is recorded so selected tests can run fastest first.
19
19
  class CoverageMap
20
- FORMAT = 8
20
+ FORMAT = 11
21
21
 
22
22
  RAN_AT_LOAD = "ran at load: the code ran while the app loaded; a forked test cannot observe a change here"
23
+ PRELOADED = "loaded before coverage started"
23
24
 
24
- attr_reader :lines, :tests, :build_seconds, :durations, :methods, :load_lines, :branches
25
+ attr_reader :lines, :tests, :build_seconds, :loaded_files, :durations, :methods, :load_lines, :load_methods,
26
+ :branches, :preloaded
25
27
 
26
28
  # lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
27
29
  # methods: { "/abs/file.rb" => { 12 => ["id1", "id2"] } } keyed by the
28
30
  # def's first line; line coverage never counts an endless def's body,
29
31
  # so methods carry those subjects (issue #107).
30
32
  # durations: { "id1" => seconds }
31
- def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, branches: {})
33
+ def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, load_methods: {},
34
+ branches: {}, loaded_files: nil, preloaded: [])
32
35
  @lines = lines
33
36
  @tests = tests
34
37
  @build_seconds = build_seconds
35
38
  @durations = durations
36
39
  @methods = methods
37
40
  @load_lines = load_lines
41
+ @load_methods = load_methods
38
42
  @branches = branches
43
+ @loaded_files = loaded_files || []
44
+ @preloaded = preloaded.to_set { |file| CoverageMap.canonical(file) }
45
+ end
46
+
47
+ # Source files the given tests touched, sorted and deduped: the
48
+ # dependency set a cached verdict must watch (#197). The inversion is
49
+ # built once per map and memoized.
50
+ def files_for(test_ids)
51
+ by_test = @files_by_test ||= invert_coverage
52
+ test_ids.flat_map { |id| by_test.fetch(id, []) }.uniq.sort
53
+ end
54
+
55
+ # { test id => [source files it touched] }, built once per map. Line
56
+ # and method coverage both count: an endless def's file has only
57
+ # method evidence (#107), and a change there must still invalidate (#344).
58
+ def invert_coverage
59
+ [lines, methods].each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |hits, acc|
60
+ hits.each { |file, line_tests| line_tests.each_value { |ids| ids.each { |id| acc[id] << file } } }
61
+ end
62
+ end
63
+
64
+ # Files that ran while the app loaded: their state precedes every
65
+ # test, so every cached verdict depends on them (#344).
66
+ def boot_files
67
+ load_lines.keys.sort
39
68
  end
40
69
 
41
70
  def tests_for(subject)
71
+ return tests_for_constant(subject) if subject.is_a?(ConstantSubject)
72
+
42
73
  tests = tests_for_lines(subject.file, subject.coverage_lines)
43
74
  return tests unless tests.empty?
44
75
 
@@ -50,10 +81,46 @@ module Saboteur
50
81
  sort_test_ids(ids)
51
82
  end
52
83
 
84
+ # A constant is observable wherever it is read: the tests that ran
85
+ # any read of it, across every file the consumer scan found (#306).
86
+ # A read inside an endless def has no line coverage, so its def's
87
+ # per-test method-call counter stands in (#319).
88
+ def tests_for_constant(subject)
89
+ ids = subject.consumer_lines.flat_map { |file, lines| tests_for_lines(file, lines) }
90
+ ids += subject.consumer_defs.flat_map do |file, def_lines|
91
+ by_method = methods[CoverageMap.canonical(file)]
92
+ by_method ? def_lines.flat_map { |line| by_method.fetch(line, []) } : []
93
+ end
94
+ sort_test_ids(ids.uniq)
95
+ end
96
+
53
97
  # Attach the parent's boot counters without losing the child's method coverage.
54
- def with_load_lines(load_lines)
98
+ def with_load_lines(load_lines, load_methods = {})
55
99
  self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
56
- methods: methods, load_lines: load_lines, branches: branches)
100
+ methods: methods, load_lines: load_lines, load_methods: load_methods, branches: branches,
101
+ loaded_files: loaded_files)
102
+ end
103
+
104
+ # Files that loaded before Coverage.start (gemspec require, RUBYOPT
105
+ # -r): they can never carry hits, so their subjects are reported as
106
+ # loaded before coverage started instead of no coverage.
107
+ def with_preloaded(paths)
108
+ self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
109
+ methods: methods, load_lines: load_lines, load_methods: load_methods, branches: branches,
110
+ loaded_files: loaded_files, preloaded: paths)
111
+ end
112
+
113
+ def preloaded?(file)
114
+ preloaded.include?(CoverageMap.canonical(file))
115
+ end
116
+
117
+ # A def the boot called (a def-time macro's class-body caller, say)
118
+ # and no test or hook called again: re-evaluating it post-boot
119
+ # changes nothing a selected test can observe, because closure
120
+ # consumers captured the pristine definition (#409).
121
+ def called_only_at_load?(file, line)
122
+ file = CoverageMap.canonical(file)
123
+ load_methods.fetch(file, []).include?(line) && methods.fetch(file, {}).fetch(line, []).empty?
57
124
  end
58
125
 
59
126
  # A requested line ran at load but no test later hit it.
@@ -69,6 +136,11 @@ module Saboteur
69
136
  # "no coverage" skip, but a subject whose lines never record at all
70
137
  # (an endless def) falls back to its method coverage (issue #107).
71
138
  def tests_for_edit(subject, line_numbers)
139
+ # An edit to a constant literal is observable at every read of the
140
+ # constant, not on the literal's own (load-time) lines: narrowing
141
+ # by edit lines is meaningless (#306).
142
+ return tests_for(subject) if subject.is_a?(ConstantSubject)
143
+
72
144
  tests = tests_for_lines(subject.file, line_numbers)
73
145
  return tests unless tests.empty?
74
146
  return [] unless tests_for_lines(subject.file, subject.coverage_lines).empty?
@@ -100,6 +172,17 @@ module Saboteur
100
172
  [lines, methods, load_lines].all? { |hits| hits.none? { |_file, values| values.any? } }
101
173
  end
102
174
 
175
+ # True when none of these files recorded a line hit, a method call, or
176
+ # a load-time hit. The recorder's map also holds the suite's spec
177
+ # files, so whole-map emptiness cannot tell a build that recorded test
178
+ # code only while the included files stayed dark.
179
+ def empty_for?(files)
180
+ wanted = files.map { |file| CoverageMap.canonical(file) }
181
+ [lines, methods, load_lines].all? do |hits|
182
+ wanted.none? { |file| hits[file]&.any? }
183
+ end
184
+ end
185
+
103
186
  # Tests that executed any of the lines, fastest first (suite order breaks
104
187
  # ties), no duplicates. With fail-fast, a mutant is killed by its
105
188
  # cheapest killing test.
@@ -124,15 +207,19 @@ module Saboteur
124
207
  hits.transform_values { |by_line| by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] } }
125
208
  end
126
209
  JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encode.call(lines),
127
- methods: encode.call(methods), load_lines: load_lines,
210
+ methods: encode.call(methods), load_lines: load_lines, load_methods: load_methods,
128
211
  branches: encode_branches(index),
129
- durations: tests.map { |id| durations[id]&.round(6) })
212
+ durations: tests.map { |id| durations[id]&.round(6) }, loaded_files: loaded_files)
130
213
  end
131
214
 
132
215
  def self.from_json(json)
133
216
  data = JSON.parse(json)
134
217
  raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
135
218
 
219
+ from_data(data)
220
+ end
221
+
222
+ def self.from_data(data)
136
223
  tests = data["tests"]
137
224
  decode = lambda do |hits|
138
225
  hits.transform_values do |by_line|
@@ -140,10 +227,14 @@ module Saboteur
140
227
  end
141
228
  end
142
229
  durations = tests.zip(data["durations"]).to_h.compact
143
- load_lines = (data["load_lines"] || {}).transform_values { |list| list.map(&:to_i) }
144
230
  new(lines: decode.call(data["lines"]), tests: tests, build_seconds: data["build_seconds"],
145
- durations: durations, methods: decode.call(data["methods"] || {}), load_lines: load_lines,
146
- branches: decode_branches(data.fetch("branches", {}), tests))
231
+ durations: durations, methods: decode.call(data["methods"] || {}),
232
+ load_lines: int_lists(data, "load_lines"), load_methods: int_lists(data, "load_methods"),
233
+ branches: decode_branches(data.fetch("branches", {}), tests), loaded_files: data["loaded_files"])
234
+ end
235
+
236
+ def self.int_lists(data, key)
237
+ (data[key] || {}).transform_values { |list| list.map(&:to_i) }
147
238
  end
148
239
 
149
240
  def self.decode_branches(branches, tests)
@@ -168,6 +259,25 @@ module Saboteur
168
259
  File.expand_path(path)
169
260
  end
170
261
 
262
+ # Every spelling the root can have in $LOADED_FEATURES: a symlink on
263
+ # the root path (macOS /var -> /private/var) gives the same directory
264
+ # both the resolved and the spelled form, and canonical() resolves a
265
+ # loaded feature only while its file exists. Match membership against
266
+ # both, each with a trailing slash so prefix siblings stay out.
267
+ def self.root_prefixes(root)
268
+ [File.realpath(root), File.expand_path(root)].uniq.map { |path| "#{path}/" }
269
+ end
270
+
271
+ # The dependency universe a cached verdict watches when coverage
272
+ # cannot speak: every project file the booted process loaded, plus
273
+ # the run's own files (#344).
274
+ def self.loaded_project_files(files, prefixes:)
275
+ $LOADED_FEATURES.filter_map do |file|
276
+ path = Text.utf8_strict(canonical(file))
277
+ path if path.end_with?(".rb") && prefixes.any? { |prefix| path.start_with?(prefix) }
278
+ end | files
279
+ end
280
+
171
281
  def self.load(path)
172
282
  from_json(Text.utf8_strict(File.read(path))) if File.file?(path)
173
283
  rescue JSON::ParserError, Error
@@ -0,0 +1,80 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Selection
6
+ # Dependency trees are never changed source (#504): a vendored bundle
7
+ # or node_modules, tracked or untracked, git-ignored or not, is
8
+ # rejected at enumeration, so --since never lists, reads or warns on
9
+ # it. The project-local bundler path (.bundle/config's BUNDLE_PATH)
10
+ # joins the shield when set - the standard CI layout vendor/bundle is
11
+ # already under vendor/. The config file itself is project
12
+ # configuration, not vendored source, and stays visible.
13
+ module DependencyShield
14
+ TREES = %w[vendor/ node_modules/].freeze
15
+ BUNDLER_CONFIG = ".bundle/config"
16
+
17
+ module_function
18
+
19
+ def reject_trees(paths, root)
20
+ bundler = bundler_path(root)
21
+ paths.reject do |path|
22
+ next false if path == BUNDLER_CONFIG
23
+
24
+ TREES.any? { |tree| path.start_with?(tree) } || (bundler && path.start_with?(bundler))
25
+ end
26
+ end
27
+
28
+ # .bundle/config's BUNDLE_PATH as a relative directory prefix, or
29
+ # nil when no project-local config sets one (an absolute path is the
30
+ # system bundle, outside the tree, so nothing to shield).
31
+ def bundler_path(root)
32
+ config = File.join(root, ".bundle", "config")
33
+ return unless File.file?(config)
34
+
35
+ line = File.foreach(config).find { |entry| entry.start_with?("BUNDLE_PATH:") }
36
+ return unless line
37
+
38
+ value = scalar(line.split(":", 2).last.strip)
39
+ return if value.empty?
40
+
41
+ # BUNDLE_PATH: . (or ./) is the project root itself - the empty
42
+ # prefix would match every path and shield the whole tree (#580) -
43
+ # and anything outside the root is the system bundle, not project
44
+ # source. The prefix is the normalized path, not the raw scalar:
45
+ # ./deps/../deps shields deps/.
46
+ base = File.expand_path(root)
47
+ expanded = File.expand_path(value, base)
48
+ return if expanded == base || !expanded.start_with?("#{base}/")
49
+
50
+ "#{expanded.delete_prefix("#{base}/")}/"
51
+ # ArgumentError too: a NUL byte in the value makes File.expand_path
52
+ # raise it, and a config that cannot be read means no shield.
53
+ rescue SystemCallError, ArgumentError
54
+ nil
55
+ end
56
+
57
+ # The scalar after "BUNDLE_PATH:": bundler writes a bare word, but
58
+ # YAML also allows it quoted - surrounding matching quotes are not
59
+ # part of the path.
60
+ def scalar(raw)
61
+ quote = raw[0]
62
+ value = if raw.length >= 2 && raw[-1] == quote && ["'", '"'].include?(quote)
63
+ raw[1..-2]
64
+ else
65
+ raw
66
+ end
67
+ strip_comment(value)
68
+ end
69
+
70
+ # Bundler::YAMLSerializer#strip_comment, which parses .bundle/config
71
+ # for Bundler itself: a # after the first character starts a comment
72
+ # even inside quotes, while a leading # is data.
73
+ def strip_comment(value)
74
+ return value unless value.include?("#") && !value.start_with?("#")
75
+
76
+ value.split("#", 2).first.strip
77
+ end
78
+ end
79
+ end
80
+ end