saboteur 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +215 -0
- data/README.md +44 -6
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +116 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +6 -1
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +29 -305
- data/lib/saboteur/config.rb +76 -7
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +24 -5
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +6 -4
- data/lib/saboteur/doctor.rb +10 -0
- data/lib/saboteur/engine/baseline_guard.rb +118 -15
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +16 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +296 -21
- data/lib/saboteur/engine/diagnostics.rb +24 -7
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +4 -1
- data/lib/saboteur/engine/load_coverage.rb +72 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +11 -3
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +10 -7
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +26 -5
- data/lib/saboteur/engine/scheduler.rb +33 -25
- data/lib/saboteur/engine/selection_announcement.rb +9 -3
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +144 -64
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +186 -32
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +44 -11
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +123 -1
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +39 -20
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +42 -8
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +145 -18
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +33 -21
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +13 -0
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +45 -5
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +150 -9
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +9 -0
- metadata +48 -16
data/lib/saboteur/gates.rb
CHANGED
|
@@ -60,59 +60,152 @@ module Saboteur
|
|
|
60
60
|
!@limits.fetch("max_errors").nil?
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
# Prints each failed gate to err
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
# Prints the gate notes and each failed gate to err. The first
|
|
64
|
+
# violation's cause is recorded for the JSON report unless the caller
|
|
65
|
+
# already has a cause that must win (record_cause: false).
|
|
66
|
+
def failed?(report, err, record_cause: true)
|
|
67
|
+
overlap_notes(report).each { |note| err.puts("saboteur: #{note}") }
|
|
66
68
|
found = violations(report)
|
|
67
69
|
found.each { |violation| err.puts("saboteur: #{violation.message}") }
|
|
68
|
-
@cause = found.first&.cause
|
|
69
|
-
|
|
70
|
+
@cause = found.first&.cause if record_cause
|
|
71
|
+
found.any?
|
|
70
72
|
end
|
|
71
73
|
|
|
72
74
|
def violations(report)
|
|
73
75
|
SETTINGS.filter_map do |key, (status, cause, flag)|
|
|
74
76
|
limit = @limits.fetch(key)
|
|
77
|
+
next if limit.nil?
|
|
78
|
+
|
|
75
79
|
count = report.counts.fetch(status)
|
|
76
|
-
next
|
|
80
|
+
next unless over?(count, limit, report.executed.size)
|
|
77
81
|
|
|
78
82
|
Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
|
|
79
83
|
end + score_gate_violations(report)
|
|
80
84
|
end
|
|
81
85
|
|
|
86
|
+
# A machine-readable summary per score gate for the JSON report: the
|
|
87
|
+
# floor, the mutants scoring under it, its score and pass/fail state,
|
|
88
|
+
# the failing subjects and the matched files. nil without score gates.
|
|
89
|
+
def score_gate_breakdowns(report)
|
|
90
|
+
return if @score_gates.empty?
|
|
91
|
+
|
|
92
|
+
assignment = gate_assignment(report)
|
|
93
|
+
@score_gates.each_with_index.map do |gate, index|
|
|
94
|
+
scored = assignment.fetch(index)
|
|
95
|
+
verdict = gate_verdict(gate, scored)
|
|
96
|
+
{ pattern: gate["pattern"], minScore: gate["min_score"], mutants: scored.size, tested: verdict[:tested],
|
|
97
|
+
score: verdict[:score]&.round(1), passed: verdict[:passed],
|
|
98
|
+
failingSubjects: failing_subjects(scored), files: gate_files(scored, report) }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
82
102
|
private
|
|
83
103
|
|
|
84
|
-
# Per-path gates (#473): each gate scores the results
|
|
85
|
-
#
|
|
86
|
-
#
|
|
104
|
+
# Per-path gates (#473): each gate scores the results assigned to it by
|
|
105
|
+
# gate_matching, with the run's own Score formula, and fails under its
|
|
106
|
+
# floor. The failure line names the matched files (at most three, then
|
|
107
|
+
# a count) and the failing subject count.
|
|
87
108
|
def score_gate_violations(report)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
109
|
+
assignment = gate_assignment(report)
|
|
110
|
+
@score_gates.each_with_index.filter_map do |gate, index|
|
|
111
|
+
scored = assignment.fetch(index)
|
|
112
|
+
verdict = gate_verdict(gate, scored)
|
|
113
|
+
next if verdict[:passed]
|
|
114
|
+
|
|
115
|
+
files = gate_files(scored, report)
|
|
116
|
+
listed = files.first(3).join(", ")
|
|
117
|
+
listed += ", +#{files.size - 3} more" if files.size > 3
|
|
118
|
+
message = "score gate #{gate['pattern']}: score #{format('%.1f', verdict[:score])} "
|
|
119
|
+
message += "below #{gate['min_score']} (#{scored.size} mutant(s) under it, "
|
|
120
|
+
message += "#{failing_subjects(scored)} failing subject(s), #{files.size} matched file(s): #{listed})"
|
|
121
|
+
Violation.new(cause: "scoreGate", message:)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
92
124
|
|
|
93
|
-
|
|
94
|
-
|
|
125
|
+
# A gate that lost subjects to a more specific gate says so once per
|
|
126
|
+
# subject, even on a passing run. Only a gate that matched nothing at
|
|
127
|
+
# all is vacuous; a fully shadowed gate reads as overlapped instead.
|
|
128
|
+
def overlap_notes(report)
|
|
129
|
+
matched, assignment = gate_matching(report)
|
|
130
|
+
@score_gates.each_with_index.filter_map do |gate, index|
|
|
131
|
+
lost = (matched[index] - assignment.fetch(index)).map(&:subject).uniq.size
|
|
132
|
+
if lost.positive?
|
|
133
|
+
next "score gate #{gate['pattern']}: #{lost} subject(s) also match a more specific gate and score there"
|
|
134
|
+
end
|
|
95
135
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
136
|
+
next if Score.of(assignment.fetch(index)).tested.positive?
|
|
137
|
+
|
|
138
|
+
# A gate that matched mutants but tested none of them (errors,
|
|
139
|
+
# timeouts, skips) passes just as vacuously as one that matched
|
|
140
|
+
# nothing; say so either way, unless a more specific gate shadows it.
|
|
141
|
+
if matched[index].empty?
|
|
142
|
+
"score gate #{gate['pattern']}: nothing executed under it, passing vacuously"
|
|
143
|
+
else
|
|
144
|
+
"score gate #{gate['pattern']}: no mutant(s) tested under it, passing vacuously"
|
|
145
|
+
end
|
|
99
146
|
end
|
|
100
147
|
end
|
|
101
148
|
|
|
102
|
-
#
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
149
|
+
# One verdict per gate, shared by the console violation and the JSON
|
|
150
|
+
# breakdown: both judge the unrounded score, so a 79.96 against an 80
|
|
151
|
+
# floor fails in both places. The one-decimal score in the JSON is
|
|
152
|
+
# display-only. A gate with nothing tested passes vacuously, score nil.
|
|
153
|
+
def gate_verdict(gate, scored)
|
|
154
|
+
breakdown = Score.of(scored)
|
|
155
|
+
return { tested: 0, passed: true } if breakdown.tested.zero?
|
|
156
|
+
|
|
157
|
+
raw = breakdown.value * 100
|
|
158
|
+
{ tested: breakdown.tested, score: raw, passed: raw >= gate["min_score"] }
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Each result scores under the single most specific matching gate: more
|
|
162
|
+
# literal (non-glob) pattern characters win, ties keep the earlier config
|
|
163
|
+
# entry (min_by returns the first minimum), so overlapping floors never
|
|
164
|
+
# double-count a mutant. Returns the per-gate matched and assigned lists.
|
|
165
|
+
def gate_matching(report)
|
|
166
|
+
matched = @score_gates.map { [] }
|
|
167
|
+
assignment = @score_gates.map { [] }
|
|
168
|
+
if @score_gates.any?
|
|
169
|
+
each_relative_result(report).each do |relative, result|
|
|
170
|
+
candidates = @score_gates.each_with_index.select { |gate, _index| gate_matches?(gate, relative) }
|
|
171
|
+
candidates.each { |candidate| matched[candidate[1]] << result }
|
|
172
|
+
winner = candidates.min_by { |gate, _index| -specificity(gate["pattern"]) }
|
|
173
|
+
assignment[winner[1]] << result if winner
|
|
174
|
+
end
|
|
107
175
|
end
|
|
176
|
+
[matched, assignment]
|
|
108
177
|
end
|
|
109
178
|
|
|
110
|
-
def
|
|
179
|
+
def gate_assignment(report)
|
|
180
|
+
gate_matching(report)[1]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# [relative path, result] pairs; the caller consumes the array, so the
|
|
184
|
+
# build cannot be swapped for a side-effect-only walk.
|
|
185
|
+
def each_relative_result(report)
|
|
111
186
|
prefix = File.join(File.realpath(report.root), "")
|
|
112
|
-
report.results.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
187
|
+
report.results.map { |result| [result.subject.file.delete_prefix(prefix), result] }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def gate_matches?(gate, relative)
|
|
191
|
+
File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Literal characters pin a gate to its files; glob metacharacters do
|
|
195
|
+
# not, and a {a,b} brace group counts once rather than per alternative,
|
|
196
|
+
# so the exact lib/saboteur/gates.rb outranks lib/saboteur/{gates,config}.rb.
|
|
197
|
+
def specificity(pattern)
|
|
198
|
+
pattern.gsub(/\{[^{}]*\}/, "x").gsub(/[*?\[\]\\]/, "").length
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def gate_files(scored, report)
|
|
202
|
+
prefix = File.join(File.realpath(report.root), "")
|
|
203
|
+
scored.map { |result| result.subject.file.delete_prefix(prefix) }.uniq.sort
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# A subject fails a gate when at least one of its mutants survived.
|
|
207
|
+
def failing_subjects(scored)
|
|
208
|
+
scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
|
|
116
209
|
end
|
|
117
210
|
|
|
118
211
|
def over?(count, limit, executed)
|
data/lib/saboteur/install.rb
CHANGED
|
@@ -12,7 +12,11 @@ module Saboteur
|
|
|
12
12
|
class Install
|
|
13
13
|
CONFIG = <<~RUBY
|
|
14
14
|
# saboteur configuration for a Rails + RSpec app. CLI flags override this file.
|
|
15
|
+
# Only the decisions a first run needs are set here; every other key keeps
|
|
16
|
+
# its default. Who changes what, key by key:
|
|
17
|
+
# https://github.com/vamsipavanmahesh/saboteur/blob/main/docs/config-keys.md
|
|
15
18
|
Saboteur.configure do |c|
|
|
19
|
+
# First-run decision: what to mutate.
|
|
16
20
|
c.include = ["app/**/*.rb"]
|
|
17
21
|
# Empty base classes: nothing useful to mutate.
|
|
18
22
|
c.exclude = [
|
|
@@ -23,7 +27,8 @@ module Saboteur
|
|
|
23
27
|
]
|
|
24
28
|
c.require = ["./config/environment"] # boots the app before specs load
|
|
25
29
|
c.rails = true # reconnect ActiveRecord after fork, per-worker databases
|
|
26
|
-
|
|
30
|
+
# First-run decision: parallelism; use 1 if your specs share external state.
|
|
31
|
+
c.jobs = 4
|
|
27
32
|
end
|
|
28
33
|
RUBY
|
|
29
34
|
|
|
@@ -62,18 +62,35 @@ module Saboteur
|
|
|
62
62
|
|
|
63
63
|
# Runs every test, calling recorder.test_started(id, file) before and
|
|
64
64
|
# recorder.test_finished(id) after each one. Used to build the coverage map.
|
|
65
|
-
|
|
65
|
+
# files, when given, limits the run to the tests defined in those spec
|
|
66
|
+
# files (one part of a partitioned build, #741); nil runs everything.
|
|
67
|
+
def run_all(_recorder, files: nil, seed: nil)
|
|
66
68
|
raise NotImplementedError
|
|
67
69
|
end
|
|
68
70
|
|
|
71
|
+
# How many tests run_all would run, over files when given (a part of
|
|
72
|
+
# a partitioned build, #741). Adapters that cannot split a build keep
|
|
73
|
+
# this refusal.
|
|
74
|
+
def example_count(files: nil)
|
|
75
|
+
raise NotImplementedError, "#{self.class} cannot count a part of the suite (files: #{files.inspect})"
|
|
76
|
+
end
|
|
77
|
+
|
|
69
78
|
# Checks the unchanged suite in a fresh child. Existing adapters can
|
|
70
79
|
# use their selected-test runner; adapters may override to retain full
|
|
71
80
|
# suite ordering and report every executed test.
|
|
72
|
-
def run_baseline(seed:)
|
|
81
|
+
def run_baseline(seed:, files: nil)
|
|
82
|
+
raise NotImplementedError, "#{self.class} cannot run a part of the suite (files: #{files.inspect})" if files
|
|
83
|
+
|
|
73
84
|
Kernel.srand(seed)
|
|
74
85
|
run(all_tests.map(&:id))
|
|
75
86
|
end
|
|
76
87
|
|
|
88
|
+
# Whether the framework picks tests from a generated status file that no
|
|
89
|
+
# inputs snapshot covers.
|
|
90
|
+
def status_file?(_root)
|
|
91
|
+
false
|
|
92
|
+
end
|
|
93
|
+
|
|
77
94
|
# Files whose content decides which tests exist (for the cache key).
|
|
78
95
|
def test_files
|
|
79
96
|
[]
|
|
@@ -17,6 +17,7 @@ module Saboteur
|
|
|
17
17
|
begin
|
|
18
18
|
@runner.setup($stderr, load_output)
|
|
19
19
|
rescue SystemExit => e
|
|
20
|
+
load_output.replay
|
|
20
21
|
# A spec file calling exit at load makes RSpec print "While
|
|
21
22
|
# loading <file> ..." (which names the file) and re-raise.
|
|
22
23
|
# Uncaught, the parent exits with the spec's status: exit(0)
|
|
@@ -24,7 +25,15 @@ module Saboteur
|
|
|
24
25
|
raise Error, "a spec file called exit while loading (status #{e.status})"
|
|
25
26
|
end
|
|
26
27
|
configuration.order = "rand:#{config.fetch('seed', 1)}"
|
|
27
|
-
|
|
28
|
+
if quit_while_loading?(world)
|
|
29
|
+
# RSpec's load dump ends in the saboteur frames the load was
|
|
30
|
+
# triggered through and a "No examples found." from the reporter;
|
|
31
|
+
# both contradict the tool's own message. Print the suite's error
|
|
32
|
+
# without them (#608).
|
|
33
|
+
$stdout.write LoadErrorReport.filter(load_output.text, verbose: config.fetch("verbose", false))
|
|
34
|
+
raise Error, load_failure_message(load_output.failed_files, load_output.text)
|
|
35
|
+
end
|
|
36
|
+
load_output.replay
|
|
28
37
|
|
|
29
38
|
refuse_unintended_focus
|
|
30
39
|
end
|
|
@@ -80,7 +89,10 @@ module Saboteur
|
|
|
80
89
|
|
|
81
90
|
# A missing rspec-core is the target's setup error, not a saboteur
|
|
82
91
|
# defect (#482): refuse with the remedy instead of crashing into the
|
|
83
|
-
# internal-error report.
|
|
92
|
+
# internal-error report. It is a usage error (#610): ConfigError maps
|
|
93
|
+
# to exit 2, and the remedy names docs/minitest.md rather than
|
|
94
|
+
# c.integration - rspec is the only integration, so pointing at it
|
|
95
|
+
# sent Minitest users down a dead end. Only the framework's own LoadError converts -
|
|
84
96
|
# matched on the failed feature, not the message, so a missing
|
|
85
97
|
# rspec/core_ext does not pose as rspec/core; a LoadError the app's
|
|
86
98
|
# files raise keeps the crash route.
|
|
@@ -89,8 +101,9 @@ module Saboteur
|
|
|
89
101
|
rescue LoadError => e
|
|
90
102
|
raise unless e.path == "rspec/core"
|
|
91
103
|
|
|
92
|
-
raise
|
|
93
|
-
|
|
104
|
+
raise ConfigError, "rspec is not in the bundle: the RSpec integration needs the rspec-core gem. " \
|
|
105
|
+
"Add it to the bundle's test group. saboteur's built-in integration supports RSpec only - " \
|
|
106
|
+
"see docs/minitest.md for other suites"
|
|
94
107
|
end
|
|
95
108
|
|
|
96
109
|
def all_tests
|
|
@@ -99,6 +112,14 @@ module Saboteur
|
|
|
99
112
|
.map { |example| TestCase.new(id: example.id, file: example.metadata[:rerun_file_path]) }
|
|
100
113
|
end
|
|
101
114
|
|
|
115
|
+
# Whether RSpec picks examples from its generated status file
|
|
116
|
+
# (spec/examples.txt) for --only-failures or --next-failure. RSpec's own
|
|
117
|
+
# parser answers, so abbreviations, SPEC_OPTS, .rspec, .rspec-local,
|
|
118
|
+
# ~/.rspec, the XDG options file and --options FILE all count.
|
|
119
|
+
def status_file?(root)
|
|
120
|
+
from_directory(root) { ::RSpec::Core::ConfigurationOptions.new(rspec_args).options[:only_failures] ? true : false }
|
|
121
|
+
end
|
|
122
|
+
|
|
102
123
|
def test_files
|
|
103
124
|
world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
|
|
104
125
|
end
|
|
@@ -118,17 +139,21 @@ module Saboteur
|
|
|
118
139
|
failure_details: listener.failure_details)
|
|
119
140
|
end
|
|
120
141
|
|
|
121
|
-
def
|
|
122
|
-
|
|
142
|
+
def example_count(files: nil)
|
|
143
|
+
world.example_count(runnable_groups(files))
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def run_all(recorder, files: nil, seed: nil)
|
|
147
|
+
configuration.order = "rand:#{seed || config.fetch('seed', 1)}"
|
|
123
148
|
configuration.fail_fast = false
|
|
124
|
-
execute(Listener.new(recorder), log_phase: true)
|
|
149
|
+
execute(Listener.new(recorder), log_phase: true, files: files)
|
|
125
150
|
end
|
|
126
151
|
|
|
127
|
-
def run_baseline(seed:)
|
|
152
|
+
def run_baseline(seed:, files: nil)
|
|
128
153
|
configuration.order = "rand:#{seed}"
|
|
129
154
|
configuration.fail_fast = false
|
|
130
155
|
listener = Listener.new
|
|
131
|
-
passed = execute(listener, log_phase: true)
|
|
156
|
+
passed = execute(listener, log_phase: true, files: files)
|
|
132
157
|
RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
|
|
133
158
|
executed_ids: listener.executed, runtime: listener.runtime,
|
|
134
159
|
failure_classes: listener.failure_classes, failure_messages: listener.failure_messages,
|
|
@@ -137,6 +162,19 @@ module Saboteur
|
|
|
137
162
|
|
|
138
163
|
SHOWN_LOAD_FAILURES = 5
|
|
139
164
|
|
|
165
|
+
# Ruby's Coverage refuses a second start, so a suite whose
|
|
166
|
+
# spec_helper auto-starts SimpleCov/SingleCov collides with
|
|
167
|
+
# saboteur's own Coverage.start and the load quits with an opaque
|
|
168
|
+
# "No examples found" (#645, real run: audited @ dbf84326). Name the
|
|
169
|
+
# suspect and both guard styles. The gem's backtrace line is the
|
|
170
|
+
# fallback: the RuntimeError's text is Coverage's own, whoever
|
|
171
|
+
# started it second.
|
|
172
|
+
COVERAGE_DOUBLE_START = /coverage measurement is already setup/i
|
|
173
|
+
# A gem backtrace frame (path:line) or the start call itself - never a
|
|
174
|
+
# bare name, which a spec FILE name like simplecov_config_spec.rb
|
|
175
|
+
# would false-positive on (Pod 3 QA probe, #645).
|
|
176
|
+
COVERAGE_GEM_FRAME = %r{(?:single_cov|simplecov)[\w.\-/]*\.rb:\d+|\b(?:SimpleCov\.start|SingleCov\.setup)}i
|
|
177
|
+
|
|
140
178
|
private
|
|
141
179
|
|
|
142
180
|
# rspec-core 3.11 added rspec_is_quitting (SyntaxError and SystemExit
|
|
@@ -151,7 +189,7 @@ module Saboteur
|
|
|
151
189
|
# printed are what a user needs. Name them and the way out: a repo
|
|
152
190
|
# whose spec tree cannot load whole (an engine's dummy app, an
|
|
153
191
|
# env-dependent integration spec) must narrow what loads (#378).
|
|
154
|
-
def load_failure_message(failed_files)
|
|
192
|
+
def load_failure_message(failed_files, captured_text = nil)
|
|
155
193
|
message = +"rspec quit while loading spec files"
|
|
156
194
|
if failed_files.any?
|
|
157
195
|
shown = failed_files.first(SHOWN_LOAD_FAILURES)
|
|
@@ -164,6 +202,18 @@ module Saboteur
|
|
|
164
202
|
'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
|
|
165
203
|
"suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
|
|
166
204
|
'this; see README "Rails engines and dummy apps".'
|
|
205
|
+
hints = EnvironmentHints.all(config.root)
|
|
206
|
+
message << "\nSuspect: #{hints.join('; ')}." if hints.any?
|
|
207
|
+
if coverage_suspect?(captured_text)
|
|
208
|
+
message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
|
|
209
|
+
"Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
|
|
210
|
+
"and see docs/recipes/README.md#simplecov."
|
|
211
|
+
end
|
|
212
|
+
message
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def coverage_suspect?(captured_text)
|
|
216
|
+
captured_text&.match?(COVERAGE_DOUBLE_START) || captured_text&.match?(COVERAGE_GEM_FRAME) || false
|
|
167
217
|
end
|
|
168
218
|
|
|
169
219
|
def refuse_unintended_focus
|
|
@@ -224,10 +274,10 @@ module Saboteur
|
|
|
224
274
|
# redirected to the phase's log: progress and failure detail land in
|
|
225
275
|
# that log, so a failed phase can be diagnosed from it. Mutant runs
|
|
226
276
|
# stay quiet.
|
|
227
|
-
def execute(listener, log_phase: false)
|
|
277
|
+
def execute(listener, log_phase: false, files: nil)
|
|
228
278
|
reporter = ::RSpec::Core::Reporter.new(configuration)
|
|
229
279
|
reporter.register_listener(listener, *Listener::EVENTS)
|
|
230
|
-
groups =
|
|
280
|
+
groups = runnable_groups(files)
|
|
231
281
|
count = world.example_count(groups)
|
|
232
282
|
if log_phase
|
|
233
283
|
logger = PhaseLogger.new($stdout, total: count)
|
|
@@ -241,6 +291,31 @@ module Saboteur
|
|
|
241
291
|
passed && !world.non_example_failure
|
|
242
292
|
end
|
|
243
293
|
|
|
294
|
+
# The top-level groups with examples to run, in the world's order. With
|
|
295
|
+
# files, only the groups a spec file in that list defines (a part of a
|
|
296
|
+
# partitioned coverage build, #741). Both sides are compared as real
|
|
297
|
+
# paths. An empty list is refused and so is a listed file that defines
|
|
298
|
+
# no group: a part that silently runs nothing would leave its tests
|
|
299
|
+
# out of the map.
|
|
300
|
+
def runnable_groups(files)
|
|
301
|
+
groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
|
|
302
|
+
return groups unless files
|
|
303
|
+
raise ArgumentError, "files must name at least one spec file" if files.empty?
|
|
304
|
+
|
|
305
|
+
wanted = files.to_set { |file| File.realpath(file) }
|
|
306
|
+
chosen = groups.select { |group| wanted.include?(real_path(group.metadata[:file_path])) }
|
|
307
|
+
missing = wanted - chosen.to_set { |group| real_path(group.metadata[:file_path]) }
|
|
308
|
+
raise ArgumentError, "no example group in: #{missing.sort.join(', ')}" unless missing.empty?
|
|
309
|
+
|
|
310
|
+
chosen
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def real_path(path)
|
|
314
|
+
File.realpath(path)
|
|
315
|
+
rescue SystemCallError
|
|
316
|
+
File.expand_path(path)
|
|
317
|
+
end
|
|
318
|
+
|
|
244
319
|
# Progress and failure detail for a phase log. Every 25 finished
|
|
245
320
|
# examples is enough to see movement on a tail; failures print RSpec's
|
|
246
321
|
# full formatted detail, and the run ends with RSpec's totals line.
|
|
@@ -293,26 +368,78 @@ module Saboteur
|
|
|
293
368
|
@captured = +""
|
|
294
369
|
end
|
|
295
370
|
|
|
371
|
+
# Buffer only: the caller decides after setup whether the captured
|
|
372
|
+
# text goes out as it is (a clean load), filtered (a load quit,
|
|
373
|
+
# #608) or verbatim on a SystemExit. Passing through during setup
|
|
374
|
+
# would print what a later filter needs to drop.
|
|
296
375
|
def write(text)
|
|
297
|
-
@captured << text
|
|
298
|
-
|
|
376
|
+
@replayed ? __getobj__.write(text) : @captured << text
|
|
377
|
+
self
|
|
299
378
|
end
|
|
300
379
|
|
|
301
380
|
def print(*args)
|
|
302
|
-
|
|
303
|
-
super
|
|
381
|
+
write(args.join)
|
|
304
382
|
end
|
|
305
383
|
|
|
306
384
|
def puts(*args)
|
|
307
|
-
|
|
308
|
-
|
|
385
|
+
write("#{args.flatten.join("\n")}\n")
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def replay
|
|
389
|
+
__getobj__.write(@captured)
|
|
390
|
+
# RSpec retains this stream for later suite-hook errors.
|
|
391
|
+
@replayed = true
|
|
309
392
|
end
|
|
310
393
|
|
|
394
|
+
def text = @captured
|
|
395
|
+
|
|
311
396
|
def failed_files
|
|
312
397
|
LOAD_FAILURE.flat_map { |pattern| @captured.scan(pattern) }.flatten
|
|
313
398
|
end
|
|
314
399
|
end
|
|
315
400
|
|
|
401
|
+
# The load dump RSpec prints when a spec file raises at load, trimmed
|
|
402
|
+
# to what the user acts on: the first error with the suite's own
|
|
403
|
+
# frames. The saboteur frames the load was triggered through and the
|
|
404
|
+
# reporter's "No examples found." (nothing ran because nothing loaded)
|
|
405
|
+
# read as saboteur crashing and contradict the tool's message; both
|
|
406
|
+
# stay under --verbose (#608).
|
|
407
|
+
class LoadErrorReport
|
|
408
|
+
HEADER = /^An error occurred while loading /
|
|
409
|
+
# The running tool's install root (a checkout or an unpacked gem),
|
|
410
|
+
# with separator. Only frames under it are saboteur's: the same
|
|
411
|
+
# lib/saboteur/ layout under ANOTHER root is the target's vendored
|
|
412
|
+
# copy or a dogfooded checkout, and those frames are the suite's
|
|
413
|
+
# (#608).
|
|
414
|
+
TOOL_ROOT = "#{File.expand_path('../../..', __dir__)}/".freeze
|
|
415
|
+
private_constant :HEADER, :TOOL_ROOT
|
|
416
|
+
|
|
417
|
+
def self.filter(text, verbose: false)
|
|
418
|
+
return text if verbose
|
|
419
|
+
|
|
420
|
+
lines = text.lines.reject { |line| line.strip == "No examples found." || saboteur_frame?(line) }
|
|
421
|
+
heads = lines.each_index.select { |index| lines[index].match?(HEADER) }
|
|
422
|
+
return lines.join if heads.size <= 1
|
|
423
|
+
|
|
424
|
+
kept = lines[0...heads[1]]
|
|
425
|
+
kept.pop while kept.last&.strip == ""
|
|
426
|
+
omitted = heads.size - 1
|
|
427
|
+
kept << "(#{omitted} more load error#{'s' if omitted > 1} omitted; rerun with --verbose for the full list)\n"
|
|
428
|
+
kept.join
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
# A backtrace frame inside the tool's install root. Relative frames
|
|
432
|
+
# (the suite's own "./spec/...") are never the tool's.
|
|
433
|
+
def self.saboteur_frame?(line)
|
|
434
|
+
line = line.strip
|
|
435
|
+
return false unless line.start_with?("# ")
|
|
436
|
+
|
|
437
|
+
path = line.sub(/\A#\s+/, "").sub(/:\d+:in .*\z/, "")
|
|
438
|
+
path.start_with?("/") && File.expand_path(path).start_with?(TOOL_ROOT)
|
|
439
|
+
end
|
|
440
|
+
private_class_method :saboteur_frame?
|
|
441
|
+
end
|
|
442
|
+
|
|
316
443
|
class Listener
|
|
317
444
|
EVENTS = %i[example_started example_finished example_failed].freeze
|
|
318
445
|
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# Records which lines ran, per file, with a TracePoint on :line (#725). A
|
|
6
|
+
# per-test recorder can drain it between tests instead of calling
|
|
7
|
+
# Coverage.result, whose read walks every file of the process. Pass `files`
|
|
8
|
+
# (canonical paths) to keep only those; nil keeps all. Not wired into the
|
|
9
|
+
# recorder yet: it matches Coverage's lines mode on saboteur's own suite
|
|
10
|
+
# line for line, but has no branches, so the recorder's branch data still
|
|
11
|
+
# needs its own design.
|
|
12
|
+
class LineHits
|
|
13
|
+
def initialize(files = nil, canonical: ->(path) { path })
|
|
14
|
+
@wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
|
|
15
|
+
@hits = new_hits
|
|
16
|
+
@trace = TracePoint.new(:line, &method(:record))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def enable
|
|
20
|
+
@trace.enable
|
|
21
|
+
self
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def disable
|
|
25
|
+
@trace.disable
|
|
26
|
+
self
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Stops recording without touching the TracePoint: toggling it costs
|
|
30
|
+
# milliseconds once an app is loaded, so a recorder pauses around its own
|
|
31
|
+
# work instead.
|
|
32
|
+
def pause
|
|
33
|
+
@paused = true
|
|
34
|
+
self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def resume
|
|
38
|
+
@paused = false
|
|
39
|
+
self
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# file => Set of line numbers since the last drain. A failure inside the
|
|
43
|
+
# tracer is raised here: raising inside the TracePoint would unwind the
|
|
44
|
+
# program being traced.
|
|
45
|
+
def drain(clear: true)
|
|
46
|
+
failure = @failure
|
|
47
|
+
@failure = nil
|
|
48
|
+
raise failure if failure
|
|
49
|
+
|
|
50
|
+
hits = @hits
|
|
51
|
+
@hits = new_hits if clear
|
|
52
|
+
hits
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def new_hits
|
|
58
|
+
Hash.new { |hash, file| hash[file] = Set.new }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A path the canonicalizer refuses is not a wanted file: a NUL byte in an
|
|
62
|
+
# eval'd file name raises ArgumentError, and a name that is not a file
|
|
63
|
+
# ("(eval at ...)") raises ENOENT from a realpath canonicalizer.
|
|
64
|
+
def wanted?(files, canonical, path)
|
|
65
|
+
files.include?(canonical.call(path))
|
|
66
|
+
rescue ArgumentError, SystemCallError, TypeError
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def record(event)
|
|
71
|
+
return if @paused
|
|
72
|
+
|
|
73
|
+
path = event.path
|
|
74
|
+
return if @wanted && !@wanted[path]
|
|
75
|
+
|
|
76
|
+
@hits[path] << event.lineno
|
|
77
|
+
rescue StandardError => e
|
|
78
|
+
@failure ||= e
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|