saboteur 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +280 -1
  3. data/README.md +635 -77
  4. data/exe/saboteur +6 -2
  5. data/lib/generators/saboteur/install/install_generator.rb +21 -0
  6. data/lib/saboteur/agent_skill.rb +46 -0
  7. data/lib/saboteur/baseline/acceptance.rb +218 -0
  8. data/lib/saboteur/baseline/command.rb +80 -0
  9. data/lib/saboteur/baseline/comparison.rb +105 -0
  10. data/lib/saboteur/baseline/completeness.rb +53 -0
  11. data/lib/saboteur/baseline/evidence.rb +114 -0
  12. data/lib/saboteur/baseline/identity.rb +78 -0
  13. data/lib/saboteur/baseline/manifest.rb +164 -0
  14. data/lib/saboteur/baseline/run_evidence.rb +117 -0
  15. data/lib/saboteur/baseline/workflow.rb +121 -0
  16. data/lib/saboteur/baseline_health.rb +1 -1
  17. data/lib/saboteur/cli/exit_status.rb +93 -0
  18. data/lib/saboteur/cli/run_reporting.rb +75 -0
  19. data/lib/saboteur/cli.rb +171 -63
  20. data/lib/saboteur/config.rb +101 -19
  21. data/lib/saboteur/config_file.rb +97 -0
  22. data/lib/saboteur/custom_map.rb +170 -0
  23. data/lib/saboteur/doctor.rb +57 -9
  24. data/lib/saboteur/engine/baseline_guard.rb +44 -5
  25. data/lib/saboteur/engine/boot.rb +9 -0
  26. data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
  27. data/lib/saboteur/engine/canary.rb +74 -0
  28. data/lib/saboteur/engine/candidate_selection.rb +18 -5
  29. data/lib/saboteur/engine/candidates.rb +59 -7
  30. data/lib/saboteur/engine/coverage_store.rb +53 -6
  31. data/lib/saboteur/engine/data_inputs.rb +43 -0
  32. data/lib/saboteur/engine/diagnostics.rb +176 -2
  33. data/lib/saboteur/engine/diff_scope.rb +18 -0
  34. data/lib/saboteur/engine/distribution.rb +54 -13
  35. data/lib/saboteur/engine/result_cache.rb +10 -5
  36. data/lib/saboteur/engine/run_fingerprint.rb +40 -10
  37. data/lib/saboteur/engine/sandbox_policy.rb +108 -0
  38. data/lib/saboteur/engine/scheduler.rb +99 -36
  39. data/lib/saboteur/engine/subject_selection.rb +24 -1
  40. data/lib/saboteur/engine/suite_files.rb +8 -1
  41. data/lib/saboteur/engine/test_guess.rb +52 -0
  42. data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
  43. data/lib/saboteur/engine/verdict_check.rb +7 -6
  44. data/lib/saboteur/engine.rb +257 -22
  45. data/lib/saboteur/execution/protocol.rb +2 -1
  46. data/lib/saboteur/execution/reevaluator.rb +79 -14
  47. data/lib/saboteur/execution/runner.rb +120 -17
  48. data/lib/saboteur/execution/sandbox.rb +271 -0
  49. data/lib/saboteur/execution/stale_captures.rb +43 -0
  50. data/lib/saboteur/execution/supervisor.rb +113 -9
  51. data/lib/saboteur/execution/worker_pool.rb +20 -4
  52. data/lib/saboteur/execution/wrapper.rb +45 -0
  53. data/lib/saboteur/frameworks/rails.rb +329 -13
  54. data/lib/saboteur/gates.rb +83 -0
  55. data/lib/saboteur/help.rb +63 -23
  56. data/lib/saboteur/ignore_list.rb +76 -17
  57. data/lib/saboteur/inline_suppression.rb +65 -0
  58. data/lib/saboteur/install.rb +132 -0
  59. data/lib/saboteur/integration/base.rb +1 -0
  60. data/lib/saboteur/integration/focus_guard.rb +83 -0
  61. data/lib/saboteur/integration/rspec.rb +68 -4
  62. data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
  63. data/lib/saboteur/mutation_source/builtin.rb +30 -5
  64. data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
  65. data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
  66. data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
  67. data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
  68. data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
  69. data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
  70. data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
  71. data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
  72. data/lib/saboteur/preview.rb +106 -0
  73. data/lib/saboteur/replay.rb +1 -1
  74. data/lib/saboteur/reporter/agent.rb +87 -0
  75. data/lib/saboteur/reporter/agent_clipping.rb +29 -0
  76. data/lib/saboteur/reporter/agent_commands.rb +67 -0
  77. data/lib/saboteur/reporter/agent_incomplete.rb +60 -0
  78. data/lib/saboteur/reporter/agent_summary.rb +137 -0
  79. data/lib/saboteur/reporter/findings.rb +12 -19
  80. data/lib/saboteur/reporter/history_paths.rb +38 -0
  81. data/lib/saboteur/reporter/history_record.rb +132 -0
  82. data/lib/saboteur/reporter/html.rb +61 -2
  83. data/lib/saboteur/reporter/mutation_text.rb +36 -0
  84. data/lib/saboteur/reporter/next_steps.rb +30 -0
  85. data/lib/saboteur/reporter/palette.rb +37 -0
  86. data/lib/saboteur/reporter/score_history.rb +123 -0
  87. data/lib/saboteur/reporter/score_table.rb +57 -0
  88. data/lib/saboteur/reporter/stryker.rb +25 -5
  89. data/lib/saboteur/reporter/survivor_hints.rb +68 -0
  90. data/lib/saboteur/reporter/survivor_list.rb +126 -0
  91. data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
  92. data/lib/saboteur/reporter/terminal.rb +157 -42
  93. data/lib/saboteur/reporter/test_matrix.rb +57 -0
  94. data/lib/saboteur/result.rb +86 -10
  95. data/lib/saboteur/score.rb +89 -0
  96. data/lib/saboteur/selection/coverage_map.rb +119 -13
  97. data/lib/saboteur/selection/git_diff.rb +28 -1
  98. data/lib/saboteur/selection/recorder.rb +55 -13
  99. data/lib/saboteur/subject.rb +40 -4
  100. data/lib/saboteur/subject_finder.rb +62 -12
  101. data/lib/saboteur/version.rb +1 -1
  102. data/lib/saboteur/working_tree.rb +56 -0
  103. data/lib/saboteur.rb +10 -0
  104. metadata +58 -2
@@ -16,21 +16,87 @@ module Saboteur
16
16
  # to every test in the file of the next test that starts. Each test's
17
17
  # duration is recorded so selected tests can run fastest first.
18
18
  class CoverageMap
19
- FORMAT = 3
19
+ FORMAT = 8
20
20
 
21
- attr_reader :lines, :tests, :build_seconds, :durations
21
+ RAN_AT_LOAD = "ran at load: the code ran while the app loaded; a forked test cannot observe a change here"
22
+
23
+ attr_reader :lines, :tests, :build_seconds, :durations, :methods, :load_lines, :branches
22
24
 
23
25
  # lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
26
+ # methods: { "/abs/file.rb" => { 12 => ["id1", "id2"] } } keyed by the
27
+ # def's first line; line coverage never counts an endless def's body,
28
+ # so methods carry those subjects (issue #107).
24
29
  # durations: { "id1" => seconds }
25
- def initialize(lines:, tests:, build_seconds: nil, durations: {})
30
+ def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, branches: {})
26
31
  @lines = lines
27
32
  @tests = tests
28
33
  @build_seconds = build_seconds
29
34
  @durations = durations
35
+ @methods = methods
36
+ @load_lines = load_lines
37
+ @branches = branches
30
38
  end
31
39
 
32
40
  def tests_for(subject)
33
- tests_for_lines(subject.file, subject.coverage_lines)
41
+ tests = tests_for_lines(subject.file, subject.coverage_lines)
42
+ return tests unless tests.empty?
43
+
44
+ by_method = methods[CoverageMap.canonical(subject.file)]
45
+ return [] unless by_method
46
+
47
+ span = subject.start_line..subject.end_line
48
+ ids = by_method.flat_map { |line, method_ids| span.cover?(line) ? method_ids : [] }.uniq
49
+ sort_test_ids(ids)
50
+ end
51
+
52
+ # Attach the parent's boot counters without losing the child's method coverage.
53
+ def with_load_lines(load_lines)
54
+ self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
55
+ methods: methods, load_lines: load_lines, branches: branches)
56
+ end
57
+
58
+ # A requested line ran at load but no test later hit it.
59
+ def ran_at_load?(file, line_numbers)
60
+ at_load = load_lines[CoverageMap.canonical(file)]
61
+ return false unless at_load
62
+
63
+ covered = lines[CoverageMap.canonical(file)] || {}
64
+ line_numbers.any? { |line| at_load.include?(line) && !covered.key?(line) }
65
+ end
66
+
67
+ # Tests for the lines of one edit. A branch no test executes keeps its
68
+ # "no coverage" skip, but a subject whose lines never record at all
69
+ # (an endless def) falls back to its method coverage (issue #107).
70
+ def tests_for_edit(subject, line_numbers)
71
+ tests = tests_for_lines(subject.file, line_numbers)
72
+ return tests unless tests.empty?
73
+ return [] unless tests_for_lines(subject.file, subject.coverage_lines).empty?
74
+
75
+ tests_for(subject)
76
+ end
77
+
78
+ # nil means Ruby supplied no arm evidence for this edit (including &&/||).
79
+ # An empty list means a known arm was never taken. Nested arms intersect.
80
+ def tests_for_branches(subject, edits)
81
+ arms = branches.fetch(CoverageMap.canonical(subject.file), {})
82
+ matching = edits.map do |edit|
83
+ first = position(subject.file_source, edit.start_byte)
84
+ last = position(subject.file_source, edit.end_byte)
85
+ selected = arms.select do |span, ids|
86
+ (span.first(2) <=> first) <= 0 && (last <=> span.last(2)) <= 0 &&
87
+ !ran_before_tests?(subject.file, span, ids)
88
+ end
89
+ selected.values.reduce { |ids, other| ids & other }
90
+ end
91
+ return nil if matching.include?(nil)
92
+
93
+ sort_test_ids(matching.flatten.uniq)
94
+ end
95
+
96
+ # True when neither boot nor tests recorded included code. Load-only
97
+ # evidence is valid, even though it supplies no judged mutation score.
98
+ def empty?
99
+ [lines, methods, load_lines].all? { |hits| hits.none? { |_file, values| values.any? } }
34
100
  end
35
101
 
36
102
  # Tests that executed any of the lines, fastest first (suite order breaks
@@ -46,19 +112,19 @@ module Saboteur
46
112
  by_line = lines[CoverageMap.canonical(file)]
47
113
  return [] unless by_line
48
114
 
49
- ids = line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq
50
- order = test_order
51
- ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
115
+ sort_test_ids(line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq)
52
116
  end
53
117
 
54
118
  # Lines refer to tests by their index in tests, which keeps the cache
55
119
  # small for suites with long test ids.
56
120
  def to_json(*)
57
121
  index = test_order
58
- encoded = lines.transform_values do |by_line|
59
- by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] }
122
+ encode = lambda do |hits|
123
+ hits.transform_values { |by_line| by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] } }
60
124
  end
61
- JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encoded,
125
+ JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encode.call(lines),
126
+ methods: encode.call(methods), load_lines: load_lines,
127
+ branches: encode_branches(index),
62
128
  durations: tests.map { |id| durations[id]&.round(6) })
63
129
  end
64
130
 
@@ -67,11 +133,22 @@ module Saboteur
67
133
  raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
68
134
 
69
135
  tests = data["tests"]
70
- lines = data["lines"].transform_values do |by_line|
71
- by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
136
+ decode = lambda do |hits|
137
+ hits.transform_values do |by_line|
138
+ by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
139
+ end
72
140
  end
73
141
  durations = tests.zip(data["durations"]).to_h.compact
74
- new(lines: lines, tests: tests, build_seconds: data["build_seconds"], durations: durations)
142
+ load_lines = (data["load_lines"] || {}).transform_values { |list| list.map(&:to_i) }
143
+ new(lines: decode.call(data["lines"]), tests: tests, build_seconds: data["build_seconds"],
144
+ durations: durations, methods: decode.call(data["methods"] || {}), load_lines: load_lines,
145
+ branches: decode_branches(data.fetch("branches", {}), tests))
146
+ end
147
+
148
+ def self.decode_branches(branches, tests)
149
+ branches.transform_values do |arms|
150
+ arms.to_h { |span, ids| [span, ids.map { |id| tests.fetch(id) }] }
151
+ end
75
152
  end
76
153
 
77
154
  # Cache key over every input that can change which test runs which line.
@@ -105,6 +182,35 @@ module Saboteur
105
182
 
106
183
  private
107
184
 
185
+ def encode_branches(index)
186
+ branches.transform_values do |arms|
187
+ arms.map { |span, ids| [span, ids.map { |id| index.fetch(id) }] }
188
+ end
189
+ end
190
+
191
+ # An arm whose body the app ran while loading (a def inside an if) has no
192
+ # per-test hits, yet tests ran the lines inside it. Only lines strictly
193
+ # between the arm's first and last line count: the boundary lines can
194
+ # also hold code outside the arm. An arm with no interior line, such as
195
+ # a one-line arm, cannot be told apart, so its empty list still means
196
+ # "not taken".
197
+ def ran_before_tests?(file, span, ids)
198
+ return false unless ids.empty?
199
+
200
+ covered = lines[CoverageMap.canonical(file)] || {}
201
+ ((span[0] + 1)...span[2]).any? { |line| covered.key?(line) }
202
+ end
203
+
204
+ def position(source, offset)
205
+ prefix = source.byteslice(0, offset).b
206
+ [prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
207
+ end
208
+
209
+ def sort_test_ids(ids)
210
+ order = test_order
211
+ ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
212
+ end
213
+
108
214
  def test_order
109
215
  @test_order ||= tests.each_with_index.to_h
110
216
  end
@@ -13,8 +13,35 @@ module Saboteur
13
13
 
14
14
  module_function
15
15
 
16
+ # Fails before any boot work when REF is not a commit here, as a usage
17
+ # error (exit 2): the name is wrong, or a shallow checkout lacks it.
18
+ def verify_ref!(ref, root:)
19
+ raise ConfigError, "--since needs a git ref, not an option: #{ref}" if ref.start_with?("-")
20
+
21
+ _, _, inside = Open3.capture3("git", "rev-parse", "--git-dir", chdir: root)
22
+ raise ConfigError, "--since needs a git repository" unless inside.success?
23
+
24
+ _, _, found = Open3.capture3("git", "rev-parse", "--verify", "--quiet", "#{ref}^{commit}", chdir: root)
25
+ return if found.success?
26
+
27
+ raise ConfigError, "--since #{ref}: not a commit in this repository " \
28
+ "(check the name; a shallow checkout may need a deeper fetch)"
29
+ end
30
+
31
+ def merge_base(ref, root:)
32
+ git(root, "merge-base", ref, "HEAD").strip
33
+ end
34
+
35
+ # The file's text at commit BASE, or nil when it did not exist there.
36
+ # path is relative to root.
37
+ def file_at(base, path, root:)
38
+ out, _, status = Open3.capture3("git", "show", "#{base}:./#{path}", chdir: root)
39
+ status.success? ? out.b : nil
40
+ end
41
+
16
42
  def changed_lines(ref, root:)
17
- base = git(root, "merge-base", ref, "HEAD").strip
43
+ verify_ref!(ref, root: root)
44
+ base = merge_base(ref, root: root)
18
45
  top = git(root, "rev-parse", "--show-toplevel").strip
19
46
  parse(git(root, "-c", "core.quotePath=false", "diff", "--unified=0", "--no-color", "--no-ext-diff", base,
20
47
  "--"), top)
@@ -12,10 +12,14 @@ module Saboteur
12
12
  def initialize(files)
13
13
  @files = Set.new(files.map { |file| CoverageMap.canonical(file) })
14
14
  @canonical = Hash.new { |hash, path| hash[path] = CoverageMap.canonical(path) }
15
- @lines = Hash.new { |hash, file| hash[file] = Hash.new { |by_line, line| by_line[line] = [] } }
15
+ @lines = hit_table
16
+ @methods = hit_table
17
+ @branches = hit_table
18
+ @outside_branches = per_file { Set.new }
16
19
  @tests = []
17
- @tests_by_file = Hash.new { |hash, file| hash[file] = [] }
18
- @outside = Hash.new { |hash, file| hash[file] = Set.new }
20
+ @tests_by_file = per_file { [] }
21
+ @outside = per_file { Set.new }
22
+ @outside_methods = per_file { Set.new }
19
23
  @durations = {}
20
24
  end
21
25
 
@@ -27,33 +31,64 @@ module Saboteur
27
31
  def test_started(id, test_file)
28
32
  @tests << id
29
33
  @tests_by_file[test_file] << id
30
- each_hit_line { |file, line| @outside[test_file] << [file, line] }
34
+ each_hit do |file, line, kind|
35
+ { line: @outside, method: @outside_methods, branch: @outside_branches }.fetch(kind)[test_file] << [file, line]
36
+ end
31
37
  @started_at = monotonic
32
38
  end
33
39
 
34
40
  def test_finished(id)
35
41
  @durations[id] = monotonic - @started_at if @started_at
36
- each_hit_line { |file, line| @lines[file][line] << id }
42
+ each_hit do |file, line, kind|
43
+ { line: @lines, method: @methods, branch: @branches }.fetch(kind)[file][line] << id
44
+ end
37
45
  end
38
46
 
39
47
  def result(build_seconds: nil)
40
- @outside.each do |test_file, hits|
41
- ids = @tests_by_file[test_file]
42
- hits.each { |file, line| @lines[file][line] |= ids }
43
- end
48
+ assign_outside(@outside, @lines)
49
+ assign_outside(@outside_methods, @methods)
50
+ assign_outside(@outside_branches, @branches)
44
51
  lines = @lines.transform_values { |by_line| by_line.transform_values(&:uniq) }
45
- CoverageMap.new(lines: lines, tests: @tests, build_seconds: build_seconds, durations: @durations)
52
+ methods = @methods.transform_values { |by_line| by_line.transform_values(&:uniq) }
53
+ CoverageMap.new(lines: lines, tests: @tests, build_seconds: build_seconds, durations: @durations,
54
+ methods: methods, branches: @branches.transform_values { |arms| arms.transform_values(&:uniq) })
46
55
  end
47
56
 
48
57
  private
49
58
 
59
+ def assign_outside(outside, table)
60
+ outside.each do |test_file, hits|
61
+ ids = @tests_by_file[test_file]
62
+ hits.each { |file, line| table[file][line] |= ids }
63
+ end
64
+ end
65
+
66
+ def record_branches(file, data)
67
+ data.fetch(:branches, {}).each do |(_kind, _id, *parent_span), arms|
68
+ arms.each do |(_arm_kind, _arm_id, *span), count|
69
+ next if span == parent_span
70
+
71
+ @branches[file][span] # Keep zero-hit arms as evidence, not as covered code.
72
+ yield span if count.positive?
73
+ end
74
+ end
75
+ end
76
+
77
+ def hit_table
78
+ Hash.new { |hash, file| hash[file] = Hash.new { |by_line, line| by_line[line] = [] } }
79
+ end
80
+
81
+ def per_file(&block)
82
+ Hash.new { |hash, file| hash[file] = block.call }
83
+ end
84
+
50
85
  def monotonic
51
86
  Process.clock_gettime(Process::CLOCK_MONOTONIC)
52
87
  end
53
88
 
54
89
  # Coverage is suspended while the delta is processed, so the recorder's
55
90
  # own work (and anything it calls) is never attributed to a test.
56
- def each_hit_line
91
+ def each_hit
57
92
  snapshot = Coverage.result(stop: false, clear: true)
58
93
  suspended = Coverage.running?
59
94
  Coverage.suspend if suspended
@@ -61,8 +96,15 @@ module Saboteur
61
96
  file = @canonical[path]
62
97
  next unless @files.include?(file)
63
98
 
64
- # Lines mode (set up by the executable); nil marks a line with no code.
65
- data[:lines].each_with_index { |count, index| yield file, index + 1 if count&.positive? }
99
+ # Lines mode (set up by the executable); nil marks a line with no
100
+ # code. Methods mode keys are [owner, name, start_line, ...]; an
101
+ # endless def's body never counts a line hit, but its calls count
102
+ # here (issue #107).
103
+ data[:lines].each_with_index { |count, index| yield file, index + 1, :line if count&.positive? }
104
+ record_branches(file, data) { |span| yield file, span, :branch }
105
+ data[:methods]&.each do |(_owner, _name, start_line, *), count|
106
+ yield file, start_line, :method if count.positive?
107
+ end
66
108
  end
67
109
  ensure
68
110
  Coverage.resume if suspended
@@ -24,10 +24,11 @@ module Saboteur
24
24
  # the latter, because the def text must then be evaluated on the singleton.
25
25
  class Subject
26
26
  attr_reader :name, :owner, :singleton, :sclass, :file, :start_byte, :end_byte,
27
- :start_line, :end_line, :nesting, :node, :file_source, :frozen_literals
27
+ :start_line, :end_line, :nesting, :node, :file_source, :frozen_literals, :sig_span
28
28
  attr_accessor :skip_reason
29
29
 
30
- def initialize(name:, owner:, singleton:, sclass:, file:, node:, file_source:, nesting:, frozen_literals:)
30
+ def initialize(name:, owner:, singleton:, sclass:, file:, node:, file_source:, nesting:, frozen_literals:,
31
+ sig_span: nil)
31
32
  @name = name
32
33
  @owner = owner
33
34
  @singleton = singleton
@@ -41,9 +42,20 @@ module Saboteur
41
42
  @end_byte = node.location.end_offset
42
43
  @start_line = node.location.start_line
43
44
  @end_line = node.location.end_line
45
+ @sig_span = sig_span
44
46
  @skip_reason = nil
45
47
  end
46
48
 
49
+ # The `sig { ... }` call right before the def, when there is one. A
50
+ # sorbet-wrapped method re-evaluates sig and def together, so the
51
+ # runtime type checks stay on for the mutant (see Execution::Wrapper).
52
+ def sig_source
53
+ return nil unless @sig_span
54
+
55
+ first, last = @sig_span
56
+ file_source.byteslice(first, last - first).force_encoding(Encoding::UTF_8)
57
+ end
58
+
47
59
  def id
48
60
  "#{owner}#{singleton ? '.' : '#'}#{name}"
49
61
  end
@@ -71,11 +83,14 @@ module Saboteur
71
83
 
72
84
  # Lines whose execution means the edited code ran: from the first line of
73
85
  # the innermost statement containing the edit to the edit's last line.
74
- # A statement's first line always records a coverage event when it runs,
75
- # so a test that never ran the edited code cannot have hit these lines.
86
+ # Pattern headers do not always record line events. Their enclosing case
87
+ # and clause body supply execution evidence instead.
76
88
  # An edit outside every statement (signature, around the def, whole
77
89
  # body) maps to the whole method.
78
90
  def coverage_lines_for(edit)
91
+ pattern_lines = pattern_coverage_lines(node.body, edit)
92
+ return pattern_lines if pattern_lines
93
+
79
94
  statement = innermost_statement(node.body, edit.start_byte, edit.end_byte)
80
95
  return coverage_lines unless statement
81
96
 
@@ -153,6 +168,27 @@ module Saboteur
153
168
  found
154
169
  end
155
170
 
171
+ def pattern_coverage_lines(child, edit, case_line = nil)
172
+ return nil unless child
173
+
174
+ case_line = child.location.start_line if child.is_a?(Prism::CaseMatchNode)
175
+ if child.is_a?(Prism::InNode) && contains_edit?(child.pattern, edit)
176
+ return [case_line, child.location.start_line, child.statements&.location&.start_line].compact.uniq
177
+ end
178
+
179
+ child.compact_child_nodes.each do |nested|
180
+ next unless contains_edit?(nested, edit)
181
+
182
+ lines = pattern_coverage_lines(nested, edit, case_line)
183
+ return lines if lines
184
+ end
185
+ nil
186
+ end
187
+
188
+ def contains_edit?(child, edit)
189
+ child.location.start_offset <= edit.start_byte && edit.end_byte <= child.location.end_offset
190
+ end
191
+
156
192
  def validate_edit!(edit, previous_start)
157
193
  unless File.expand_path(edit.file) == File.expand_path(file)
158
194
  raise ArgumentError, "edit for #{edit.file} applied to subject in #{file}"
@@ -9,13 +9,21 @@ module Saboteur
9
9
  #
10
10
  # Included: instance methods and `def self.x` inside a class or module,
11
11
  # defs inside `class << self`, and defs passed to a receiverless call such
12
- # as `private def x`. Skipped: top-level defs, defs nested in defs, defs in
13
- # blocks (`Class.new do`, `included do`), and `def obj.x` for obj != self.
12
+ # as `private def x`. Skipped: top-level defs, defs nested in defs, and
13
+ # `def obj.x` for obj != self. Defs in blocks (`Class.new do`, `included do`)
14
+ # are found but marked skipped ("defined inside a block"), so the report
15
+ # names them instead of leaving them out. `X = Data.define`, `Struct.new`,
16
+ # `Class.new` or `Module.new` with a block names X as the owner; any other
17
+ # block, and a `A::X = ...` target, keeps the enclosing owner or the
18
+ # placeholder.
14
19
  class SubjectFinder
15
20
  Parsed = Data.define(:subjects, :errors)
21
+ ANONYMOUS_OWNER = "(anonymous block)"
22
+ CLASS_FACTORIES = { Data: :define, Struct: :new, Class: :new, Module: :new }.freeze
16
23
 
17
- def call(file)
18
- source = File.binread(file)
24
+ # source: the text to parse when it is not the file on disk (the file as
25
+ # it was at an earlier commit).
26
+ def call(file, source: File.binread(file))
19
27
  result = Prism.parse(source, filepath: file)
20
28
  return Parsed.new(subjects: [], errors: result.errors.map(&:message)) if result.failure?
21
29
 
@@ -34,23 +42,53 @@ module Saboteur
34
42
  end
35
43
  end
36
44
 
37
- def walk(node, nesting, out, **ctx)
45
+ def walk(node, nesting, out, previous: nil, **ctx)
46
+ return if node.nil?
47
+
38
48
  case node
39
- when nil, Prism::BlockNode, Prism::LambdaNode
40
- nil
49
+ when Prism::BlockNode, Prism::LambdaNode
50
+ walk(node.body, nesting, out, **ctx, in_block: true)
51
+ when Prism::ConstantWriteNode
52
+ walk_constant_assignment(node, nesting, out, **ctx)
41
53
  when Prism::ModuleNode
42
54
  walk(node.body, nesting + [Scope.new(kind: :module, path: node.constant_path.slice)], out, **ctx)
43
55
  when Prism::ClassNode
44
56
  walk(node.body, nesting + [Scope.new(kind: :class, path: node.constant_path.slice)], out, **ctx)
45
57
  when Prism::SingletonClassNode
46
58
  walk_singleton_class(node, nesting, out, **ctx)
59
+ when Prism::StatementsNode
60
+ previous_child = nil
61
+ node.body.each do |child|
62
+ walk(child, nesting, out, previous: previous_child, **ctx)
63
+ previous_child = child
64
+ end
47
65
  when Prism::DefNode
48
- add_subject(node, nesting, out, **ctx)
66
+ add_subject(node, nesting, out, previous: previous, **ctx)
49
67
  else
50
68
  node.compact_child_nodes.each { |child| walk(child, nesting, out, **ctx) }
51
69
  end
52
70
  end
53
71
 
72
+ # `Point = Data.define(:x) do ... end`: the defs belong to Point. Only the
73
+ # four core factories count, called on the bare constant; any other call
74
+ # that takes a block says nothing about the class it returns.
75
+ def walk_constant_assignment(node, nesting, out, **ctx)
76
+ call = node.value
77
+ return node.compact_child_nodes.each { |child| walk(child, nesting, out, **ctx) } unless class_factory?(call)
78
+
79
+ # The receiver and arguments stay in the enclosing scope; only the
80
+ # factory's own block belongs to the new class.
81
+ (call.compact_child_nodes - [call.block]).each { |child| walk(child, nesting, out, **ctx) }
82
+ walk(call.block, nesting + [Scope.new(kind: :class, path: node.name.to_s)], out, **ctx)
83
+ end
84
+
85
+ def class_factory?(call)
86
+ return false unless call.is_a?(Prism::CallNode) && call.block.is_a?(Prism::BlockNode)
87
+ return false unless call.receiver.is_a?(Prism::ConstantReadNode)
88
+
89
+ CLASS_FACTORIES[call.receiver.name] == call.name
90
+ end
91
+
54
92
  def walk_singleton_class(node, nesting, out, **ctx)
55
93
  return unless node.expression.is_a?(Prism::SelfNode)
56
94
  return if nesting.empty? || nesting.last.sclass?
@@ -58,18 +96,30 @@ module Saboteur
58
96
  walk(node.body, nesting + [Scope.new(kind: :sclass, path: "self")], out, **ctx)
59
97
  end
60
98
 
61
- def add_subject(node, nesting, out, file:, source:, frozen:)
99
+ def add_subject(node, nesting, out, file:, source:, frozen:, previous: nil, in_block: false)
62
100
  in_sclass = nesting.last&.sclass? || false
63
101
  # `def self.x` inside `class << self`, and `def obj.x`, are not subjects.
64
102
  return unless node.receiver.nil? || (node.receiver.is_a?(Prism::SelfNode) && !in_sclass)
65
103
 
66
104
  singleton = in_sclass || !node.receiver.nil?
67
105
  owner = owner_path(nesting)
106
+ owner ||= ANONYMOUS_OWNER if in_block
68
107
  return unless owner
69
108
 
70
- out << Subject.new(name: node.name, owner: owner, singleton: singleton, sclass: in_sclass,
71
- file: file, node: node, file_source: source, nesting: nesting,
72
- frozen_literals: frozen)
109
+ subject = Subject.new(name: node.name, owner: owner, singleton: singleton, sclass: in_sclass,
110
+ file: file, node: node, file_source: source, nesting: nesting,
111
+ frozen_literals: frozen, sig_span: sig_span(previous))
112
+ subject.skip_reason = "defined inside a block" if in_block
113
+ out << subject
114
+ end
115
+
116
+ # The byte span of a `sig { ... }` or `T.sig { ... }` call on the lines
117
+ # right before the def, when there is one.
118
+ def sig_span(previous)
119
+ return nil unless previous.is_a?(Prism::CallNode) && previous.name == :sig && previous.block
120
+ return nil unless previous.receiver.nil? || previous.receiver.slice == "T"
121
+
122
+ [previous.location.start_offset, previous.location.end_offset]
73
123
  end
74
124
 
75
125
  # Lexical path of the enclosing class/module. `class ::Foo` restarts at the top.
@@ -2,5 +2,5 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  module Saboteur
5
- VERSION = "0.3.0"
5
+ VERSION = "0.5.0"
6
6
  end
@@ -0,0 +1,56 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "open3"
5
+
6
+ module Saboteur
7
+ # The state of the working source a run looked at: the commit it was on and
8
+ # (the full SHA) and whether files differed from it, tracked changes and each
9
+ # untracked file alike.
10
+ # Saboteur's own output (cache and reports) is passed in as ignored paths, so a
11
+ # report from an earlier run does not make the tree look dirty. Without git, or
12
+ # outside a repository, the state is unknown.
13
+ class WorkingTree
14
+ State = Data.define(:head, :dirty, :changed_files) do
15
+ def label
16
+ return "unknown" if dirty.nil?
17
+
18
+ dirty ? "dirty" : "clean"
19
+ end
20
+
21
+ # Shown before a run: mutants run your code and can change the tree.
22
+ def warning
23
+ return unless dirty
24
+
25
+ "saboteur: warning: #{changed_files} uncommitted change(s) in the working tree. " \
26
+ "Mutants run your code and could alter or delete files; run on a clean, committed tree or a CI checkout."
27
+ end
28
+ end
29
+
30
+ UNKNOWN = State.new(head: nil, dirty: nil, changed_files: nil)
31
+
32
+ # ignored: paths relative to root that are not source (saboteur's cache and reports).
33
+ def initialize(root, ignored: [])
34
+ @root = root
35
+ @ignored = ignored
36
+ end
37
+
38
+ def state
39
+ head, head_status = Open3.capture2e("git", "rev-parse", "HEAD", chdir: @root)
40
+ changes, changes_status = Open3.capture2e(*status_command, chdir: @root)
41
+ return UNKNOWN unless head_status.success? && changes_status.success?
42
+
43
+ count = changes.lines.size
44
+ State.new(head: head.strip, dirty: count.positive?, changed_files: count)
45
+ rescue SystemCallError
46
+ UNKNOWN
47
+ end
48
+
49
+ private
50
+
51
+ def status_command
52
+ excluded = @ignored.map { |path| ":(exclude)#{path}" }
53
+ ["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excluded]
54
+ end
55
+ end
56
+ end
data/lib/saboteur.rb CHANGED
@@ -9,6 +9,9 @@ module Saboteur
9
9
  # Invalid configuration or input files (exit code 2).
10
10
  class ConfigError < Error; end
11
11
 
12
+ # Files vanished from the working tree while mutants ran (exit code 3).
13
+ class CanaryTripped < Error; end
14
+
12
15
  class << self
13
16
  # Set while hook files from config load, so they can call Saboteur.hook.
14
17
  attr_accessor :current_hooks
@@ -36,18 +39,25 @@ require_relative "saboteur/execution/process_table"
36
39
  require_relative "saboteur/execution/process_tree"
37
40
  require_relative "saboteur/execution/supervisor"
38
41
  require_relative "saboteur/execution/reevaluator"
42
+ require_relative "saboteur/execution/wrapper"
39
43
  require_relative "saboteur/execution/runner"
40
44
  require_relative "saboteur/execution/worker_pool"
41
45
  require_relative "saboteur/execution/fork_readiness"
46
+ require_relative "saboteur/execution/sandbox"
42
47
  require_relative "saboteur/selection/coverage_map"
43
48
  require_relative "saboteur/selection/recorder"
44
49
  require_relative "saboteur/selection/git_diff"
45
50
  require_relative "saboteur/result"
51
+ require_relative "saboteur/custom_map"
52
+ require_relative "saboteur/gates"
46
53
  require_relative "saboteur/config"
47
54
  require_relative "saboteur/ignore_list"
48
55
  require_relative "saboteur/frameworks/rails"
49
56
  require_relative "saboteur/reporter/diff"
50
57
  require_relative "saboteur/reporter/findings"
58
+ require_relative "saboteur/reporter/agent"
59
+ require_relative "saboteur/working_tree"
60
+ require_relative "saboteur/agent_skill"
51
61
  require_relative "saboteur/reporter/terminal"
52
62
  require_relative "saboteur/reporter/stryker"
53
63
  require_relative "saboteur/reporter/html"