saboteur 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +100 -0
- data/README.md +10 -0
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +3 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +3 -0
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli.rb +29 -307
- data/lib/saboteur/config.rb +72 -6
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +20 -12
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +1 -1
- data/lib/saboteur/doctor.rb +6 -0
- data/lib/saboteur/engine/baseline_guard.rb +50 -13
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +4 -0
- data/lib/saboteur/engine/candidate_selection.rb +15 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +275 -22
- data/lib/saboteur/engine/diagnostics.rb +23 -6
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +1 -0
- data/lib/saboteur/engine/load_coverage.rb +54 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +7 -7
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +5 -5
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +5 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +1 -0
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +33 -37
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +82 -18
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +41 -9
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +105 -1
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/sandbox.rb +30 -17
- data/lib/saboteur/execution/supervisor.rb +40 -7
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +46 -7
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/builtin.rb +32 -22
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +9 -1
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +29 -4
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +136 -10
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +8 -0
- metadata +44 -2
|
@@ -91,15 +91,25 @@ module Saboteur
|
|
|
91
91
|
end
|
|
92
92
|
|
|
93
93
|
def distribute(items, order, &work)
|
|
94
|
+
@sweep_error = nil
|
|
95
|
+
# Explicit instead of $ERROR_INFO: the sweep error is raised only
|
|
96
|
+
# when the run itself completed without an error to propagate.
|
|
97
|
+
raised = true
|
|
94
98
|
results = Array.new(items.size)
|
|
95
99
|
queue = order.dup
|
|
96
100
|
workers = []
|
|
97
101
|
[@jobs, items.size].min.times { |number| workers << start(number, items, workers, &work) }
|
|
98
102
|
workers.each { |worker| assign(worker, queue) }
|
|
99
103
|
collect(workers, queue, results, items, &work) until workers.empty?
|
|
104
|
+
raised = false
|
|
100
105
|
results
|
|
101
106
|
ensure
|
|
102
107
|
workers&.each { |worker| stop(worker) }
|
|
108
|
+
# Containment first, honesty after (#762): every worker's stop ran
|
|
109
|
+
# above, so a blind-table sweep no longer aborts the registry; but a
|
|
110
|
+
# skipped orphan sweep must not read as a clean run either. The
|
|
111
|
+
# run's own error wins when one is already propagating.
|
|
112
|
+
raise @sweep_error if @sweep_error && !raised
|
|
103
113
|
end
|
|
104
114
|
|
|
105
115
|
# Waits up to a second for results, then checks for workers that died
|
|
@@ -174,11 +184,18 @@ module Saboteur
|
|
|
174
184
|
|
|
175
185
|
# Reads what a dead worker left in its pipe: a last result, or why it
|
|
176
186
|
# crashed.
|
|
177
|
-
|
|
187
|
+
# report: false while stopping - the run is unwinding, and re-invoking
|
|
188
|
+
# the arrival callback from cleanup would let its raise abort the
|
|
189
|
+
# remaining stops and leak their live entries (the CI flake).
|
|
190
|
+
def drain(worker, results, report: true)
|
|
178
191
|
pending = worker.results.read_nonblock(1 << 20, exception: false)
|
|
179
192
|
return unless pending.is_a?(String)
|
|
180
193
|
|
|
181
|
-
pending.each_line
|
|
194
|
+
pending.each_line do |line|
|
|
195
|
+
next unless line.end_with?("\n")
|
|
196
|
+
|
|
197
|
+
report ? take_and_report(worker, line, results) : take(worker, line, results)
|
|
198
|
+
end
|
|
182
199
|
rescue IOError
|
|
183
200
|
nil
|
|
184
201
|
end
|
|
@@ -331,6 +348,33 @@ module Saboteur
|
|
|
331
348
|
{ "error" => "#{e.class}: #{e.message}" }
|
|
332
349
|
end
|
|
333
350
|
|
|
351
|
+
# A blind process table (both backends down, as in #762) loses the
|
|
352
|
+
# table-driven orphan sweep, but it must not skip ending the worker
|
|
353
|
+
# itself - wait2 has no timeout, so kill_all was the only thing
|
|
354
|
+
# stopping a wedged one. Kill its group and pid directly, remember the
|
|
355
|
+
# error for distribute to raise once every worker is stopped, and never
|
|
356
|
+
# let the sweep abort the remaining stops.
|
|
357
|
+
def sweep(tree, worker)
|
|
358
|
+
tree&.kill_all
|
|
359
|
+
rescue SystemCallError, Fiddle::DLError => e
|
|
360
|
+
@sweep_error ||= e
|
|
361
|
+
# The group kill also ends orphaned grandchildren and is always
|
|
362
|
+
# safe: a process group id cannot be recycled while any member
|
|
363
|
+
# lives. The direct kill is only for a wedged worker: once reaped,
|
|
364
|
+
# its pid can already belong to an unrelated process.
|
|
365
|
+
kill_hard(worker.pid, direct: !reaped?(worker))
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def kill_hard(pid, direct:)
|
|
369
|
+
targets = [-pid]
|
|
370
|
+
targets << pid if direct
|
|
371
|
+
targets.each do |target|
|
|
372
|
+
Process.kill("KILL", target)
|
|
373
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
374
|
+
nil
|
|
375
|
+
end
|
|
376
|
+
end
|
|
377
|
+
|
|
334
378
|
# Closing the command pipe ends the worker's loop; anything it left
|
|
335
379
|
# running is killed with its tree.
|
|
336
380
|
def stop(worker)
|
|
@@ -338,9 +382,9 @@ module Saboteur
|
|
|
338
382
|
tree = Supervisor.live.delete(worker.pid)
|
|
339
383
|
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5
|
|
340
384
|
sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
341
|
-
tree
|
|
385
|
+
sweep(tree, worker)
|
|
342
386
|
worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
|
|
343
|
-
drain(worker, {})
|
|
387
|
+
drain(worker, {}, report: false)
|
|
344
388
|
worker.results.close unless worker.results.closed?
|
|
345
389
|
rescue Errno::ECHILD
|
|
346
390
|
nil
|
|
@@ -18,9 +18,6 @@ module Saboteur
|
|
|
18
18
|
# forking, and drops database connections in the parent so each child opens
|
|
19
19
|
# its own. Prints host names only, never credentials.
|
|
20
20
|
class Rails
|
|
21
|
-
# How many blocking session names the template-copy error lists.
|
|
22
|
-
MAX_BLOCKER_NAMES = 5
|
|
23
|
-
|
|
24
21
|
# sqlite3-ruby's Backup#step returns raw SQLite status codes instead
|
|
25
22
|
# of raising, and finish discards its own: without an explicit check
|
|
26
23
|
# a partial destination would pass as a copy.
|
|
@@ -362,9 +359,9 @@ module Saboteur
|
|
|
362
359
|
SQL
|
|
363
360
|
return if blockers.empty?
|
|
364
361
|
|
|
365
|
-
names = blockers.map { |app| app.to_s.empty? ? "unnamed" : app }
|
|
362
|
+
names = Text.name_list(blockers.map { |app| app.to_s.empty? ? "unnamed" : app })
|
|
366
363
|
raise Error, "cannot copy the test database #{database}: #{blockers.size} other session(s) connected " \
|
|
367
|
-
"(#{names
|
|
364
|
+
"(#{names}). " \
|
|
368
365
|
"Stop other test runs, consoles or Spring, then run saboteur again."
|
|
369
366
|
end
|
|
370
367
|
|
data/lib/saboteur/gates.rb
CHANGED
|
@@ -60,59 +60,152 @@ module Saboteur
|
|
|
60
60
|
!@limits.fetch("max_errors").nil?
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
# Prints each failed gate to err
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
# Prints the gate notes and each failed gate to err. The first
|
|
64
|
+
# violation's cause is recorded for the JSON report unless the caller
|
|
65
|
+
# already has a cause that must win (record_cause: false).
|
|
66
|
+
def failed?(report, err, record_cause: true)
|
|
67
|
+
overlap_notes(report).each { |note| err.puts("saboteur: #{note}") }
|
|
66
68
|
found = violations(report)
|
|
67
69
|
found.each { |violation| err.puts("saboteur: #{violation.message}") }
|
|
68
|
-
@cause = found.first&.cause
|
|
69
|
-
|
|
70
|
+
@cause = found.first&.cause if record_cause
|
|
71
|
+
found.any?
|
|
70
72
|
end
|
|
71
73
|
|
|
72
74
|
def violations(report)
|
|
73
75
|
SETTINGS.filter_map do |key, (status, cause, flag)|
|
|
74
76
|
limit = @limits.fetch(key)
|
|
77
|
+
next if limit.nil?
|
|
78
|
+
|
|
75
79
|
count = report.counts.fetch(status)
|
|
76
|
-
next
|
|
80
|
+
next unless over?(count, limit, report.executed.size)
|
|
77
81
|
|
|
78
82
|
Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
|
|
79
83
|
end + score_gate_violations(report)
|
|
80
84
|
end
|
|
81
85
|
|
|
86
|
+
# A machine-readable summary per score gate for the JSON report: the
|
|
87
|
+
# floor, the mutants scoring under it, its score and pass/fail state,
|
|
88
|
+
# the failing subjects and the matched files. nil without score gates.
|
|
89
|
+
def score_gate_breakdowns(report)
|
|
90
|
+
return if @score_gates.empty?
|
|
91
|
+
|
|
92
|
+
assignment = gate_assignment(report)
|
|
93
|
+
@score_gates.each_with_index.map do |gate, index|
|
|
94
|
+
scored = assignment.fetch(index)
|
|
95
|
+
verdict = gate_verdict(gate, scored)
|
|
96
|
+
{ pattern: gate["pattern"], minScore: gate["min_score"], mutants: scored.size, tested: verdict[:tested],
|
|
97
|
+
score: verdict[:score]&.round(1), passed: verdict[:passed],
|
|
98
|
+
failingSubjects: failing_subjects(scored), files: gate_files(scored, report) }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
82
102
|
private
|
|
83
103
|
|
|
84
|
-
# Per-path gates (#473): each gate scores the results
|
|
85
|
-
#
|
|
86
|
-
#
|
|
104
|
+
# Per-path gates (#473): each gate scores the results assigned to it by
|
|
105
|
+
# gate_matching, with the run's own Score formula, and fails under its
|
|
106
|
+
# floor. The failure line names the matched files (at most three, then
|
|
107
|
+
# a count) and the failing subject count.
|
|
87
108
|
def score_gate_violations(report)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
109
|
+
assignment = gate_assignment(report)
|
|
110
|
+
@score_gates.each_with_index.filter_map do |gate, index|
|
|
111
|
+
scored = assignment.fetch(index)
|
|
112
|
+
verdict = gate_verdict(gate, scored)
|
|
113
|
+
next if verdict[:passed]
|
|
114
|
+
|
|
115
|
+
files = gate_files(scored, report)
|
|
116
|
+
listed = files.first(3).join(", ")
|
|
117
|
+
listed += ", +#{files.size - 3} more" if files.size > 3
|
|
118
|
+
message = "score gate #{gate['pattern']}: score #{format('%.1f', verdict[:score])} "
|
|
119
|
+
message += "below #{gate['min_score']} (#{scored.size} mutant(s) under it, "
|
|
120
|
+
message += "#{failing_subjects(scored)} failing subject(s), #{files.size} matched file(s): #{listed})"
|
|
121
|
+
Violation.new(cause: "scoreGate", message:)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
92
124
|
|
|
93
|
-
|
|
94
|
-
|
|
125
|
+
# A gate that lost subjects to a more specific gate says so once per
|
|
126
|
+
# subject, even on a passing run. Only a gate that matched nothing at
|
|
127
|
+
# all is vacuous; a fully shadowed gate reads as overlapped instead.
|
|
128
|
+
def overlap_notes(report)
|
|
129
|
+
matched, assignment = gate_matching(report)
|
|
130
|
+
@score_gates.each_with_index.filter_map do |gate, index|
|
|
131
|
+
lost = (matched[index] - assignment.fetch(index)).map(&:subject).uniq.size
|
|
132
|
+
if lost.positive?
|
|
133
|
+
next "score gate #{gate['pattern']}: #{lost} subject(s) also match a more specific gate and score there"
|
|
134
|
+
end
|
|
95
135
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
136
|
+
next if Score.of(assignment.fetch(index)).tested.positive?
|
|
137
|
+
|
|
138
|
+
# A gate that matched mutants but tested none of them (errors,
|
|
139
|
+
# timeouts, skips) passes just as vacuously as one that matched
|
|
140
|
+
# nothing; say so either way, unless a more specific gate shadows it.
|
|
141
|
+
if matched[index].empty?
|
|
142
|
+
"score gate #{gate['pattern']}: nothing executed under it, passing vacuously"
|
|
143
|
+
else
|
|
144
|
+
"score gate #{gate['pattern']}: no mutant(s) tested under it, passing vacuously"
|
|
145
|
+
end
|
|
99
146
|
end
|
|
100
147
|
end
|
|
101
148
|
|
|
102
|
-
#
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
149
|
+
# One verdict per gate, shared by the console violation and the JSON
|
|
150
|
+
# breakdown: both judge the unrounded score, so a 79.96 against an 80
|
|
151
|
+
# floor fails in both places. The one-decimal score in the JSON is
|
|
152
|
+
# display-only. A gate with nothing tested passes vacuously, score nil.
|
|
153
|
+
def gate_verdict(gate, scored)
|
|
154
|
+
breakdown = Score.of(scored)
|
|
155
|
+
return { tested: 0, passed: true } if breakdown.tested.zero?
|
|
156
|
+
|
|
157
|
+
raw = breakdown.value * 100
|
|
158
|
+
{ tested: breakdown.tested, score: raw, passed: raw >= gate["min_score"] }
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Each result scores under the single most specific matching gate: more
|
|
162
|
+
# literal (non-glob) pattern characters win, ties keep the earlier config
|
|
163
|
+
# entry (min_by returns the first minimum), so overlapping floors never
|
|
164
|
+
# double-count a mutant. Returns the per-gate matched and assigned lists.
|
|
165
|
+
def gate_matching(report)
|
|
166
|
+
matched = @score_gates.map { [] }
|
|
167
|
+
assignment = @score_gates.map { [] }
|
|
168
|
+
if @score_gates.any?
|
|
169
|
+
each_relative_result(report).each do |relative, result|
|
|
170
|
+
candidates = @score_gates.each_with_index.select { |gate, _index| gate_matches?(gate, relative) }
|
|
171
|
+
candidates.each { |candidate| matched[candidate[1]] << result }
|
|
172
|
+
winner = candidates.min_by { |gate, _index| -specificity(gate["pattern"]) }
|
|
173
|
+
assignment[winner[1]] << result if winner
|
|
174
|
+
end
|
|
107
175
|
end
|
|
176
|
+
[matched, assignment]
|
|
108
177
|
end
|
|
109
178
|
|
|
110
|
-
def
|
|
179
|
+
def gate_assignment(report)
|
|
180
|
+
gate_matching(report)[1]
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# [relative path, result] pairs; the caller consumes the array, so the
|
|
184
|
+
# build cannot be swapped for a side-effect-only walk.
|
|
185
|
+
def each_relative_result(report)
|
|
111
186
|
prefix = File.join(File.realpath(report.root), "")
|
|
112
|
-
report.results.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
187
|
+
report.results.map { |result| [result.subject.file.delete_prefix(prefix), result] }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def gate_matches?(gate, relative)
|
|
191
|
+
File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Literal characters pin a gate to its files; glob metacharacters do
|
|
195
|
+
# not, and a {a,b} brace group counts once rather than per alternative,
|
|
196
|
+
# so the exact lib/saboteur/gates.rb outranks lib/saboteur/{gates,config}.rb.
|
|
197
|
+
def specificity(pattern)
|
|
198
|
+
pattern.gsub(/\{[^{}]*\}/, "x").gsub(/[*?\[\]\\]/, "").length
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def gate_files(scored, report)
|
|
202
|
+
prefix = File.join(File.realpath(report.root), "")
|
|
203
|
+
scored.map { |result| result.subject.file.delete_prefix(prefix) }.uniq.sort
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# A subject fails a gate when at least one of its mutants survived.
|
|
207
|
+
def failing_subjects(scored)
|
|
208
|
+
scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
|
|
116
209
|
end
|
|
117
210
|
|
|
118
211
|
def over?(count, limit, executed)
|
data/lib/saboteur/install.rb
CHANGED
|
@@ -12,7 +12,11 @@ module Saboteur
|
|
|
12
12
|
class Install
|
|
13
13
|
CONFIG = <<~RUBY
|
|
14
14
|
# saboteur configuration for a Rails + RSpec app. CLI flags override this file.
|
|
15
|
+
# Only the decisions a first run needs are set here; every other key keeps
|
|
16
|
+
# its default. Who changes what, key by key:
|
|
17
|
+
# https://github.com/vamsipavanmahesh/saboteur/blob/main/docs/config-keys.md
|
|
15
18
|
Saboteur.configure do |c|
|
|
19
|
+
# First-run decision: what to mutate.
|
|
16
20
|
c.include = ["app/**/*.rb"]
|
|
17
21
|
# Empty base classes: nothing useful to mutate.
|
|
18
22
|
c.exclude = [
|
|
@@ -23,7 +27,8 @@ module Saboteur
|
|
|
23
27
|
]
|
|
24
28
|
c.require = ["./config/environment"] # boots the app before specs load
|
|
25
29
|
c.rails = true # reconnect ActiveRecord after fork, per-worker databases
|
|
26
|
-
|
|
30
|
+
# First-run decision: parallelism; use 1 if your specs share external state.
|
|
31
|
+
c.jobs = 4
|
|
27
32
|
end
|
|
28
33
|
RUBY
|
|
29
34
|
|
|
@@ -62,18 +62,35 @@ module Saboteur
|
|
|
62
62
|
|
|
63
63
|
# Runs every test, calling recorder.test_started(id, file) before and
|
|
64
64
|
# recorder.test_finished(id) after each one. Used to build the coverage map.
|
|
65
|
-
|
|
65
|
+
# files, when given, limits the run to the tests defined in those spec
|
|
66
|
+
# files (one part of a partitioned build, #741); nil runs everything.
|
|
67
|
+
def run_all(_recorder, files: nil, seed: nil)
|
|
66
68
|
raise NotImplementedError
|
|
67
69
|
end
|
|
68
70
|
|
|
71
|
+
# How many tests run_all would run, over files when given (a part of
|
|
72
|
+
# a partitioned build, #741). Adapters that cannot split a build keep
|
|
73
|
+
# this refusal.
|
|
74
|
+
def example_count(files: nil)
|
|
75
|
+
raise NotImplementedError, "#{self.class} cannot count a part of the suite (files: #{files.inspect})"
|
|
76
|
+
end
|
|
77
|
+
|
|
69
78
|
# Checks the unchanged suite in a fresh child. Existing adapters can
|
|
70
79
|
# use their selected-test runner; adapters may override to retain full
|
|
71
80
|
# suite ordering and report every executed test.
|
|
72
|
-
def run_baseline(seed:)
|
|
81
|
+
def run_baseline(seed:, files: nil)
|
|
82
|
+
raise NotImplementedError, "#{self.class} cannot run a part of the suite (files: #{files.inspect})" if files
|
|
83
|
+
|
|
73
84
|
Kernel.srand(seed)
|
|
74
85
|
run(all_tests.map(&:id))
|
|
75
86
|
end
|
|
76
87
|
|
|
88
|
+
# Whether the framework picks tests from a generated status file that no
|
|
89
|
+
# inputs snapshot covers.
|
|
90
|
+
def status_file?(_root)
|
|
91
|
+
false
|
|
92
|
+
end
|
|
93
|
+
|
|
77
94
|
# Files whose content decides which tests exist (for the cache key).
|
|
78
95
|
def test_files
|
|
79
96
|
[]
|
|
@@ -112,6 +112,14 @@ module Saboteur
|
|
|
112
112
|
.map { |example| TestCase.new(id: example.id, file: example.metadata[:rerun_file_path]) }
|
|
113
113
|
end
|
|
114
114
|
|
|
115
|
+
# Whether RSpec picks examples from its generated status file
|
|
116
|
+
# (spec/examples.txt) for --only-failures or --next-failure. RSpec's own
|
|
117
|
+
# parser answers, so abbreviations, SPEC_OPTS, .rspec, .rspec-local,
|
|
118
|
+
# ~/.rspec, the XDG options file and --options FILE all count.
|
|
119
|
+
def status_file?(root)
|
|
120
|
+
from_directory(root) { ::RSpec::Core::ConfigurationOptions.new(rspec_args).options[:only_failures] ? true : false }
|
|
121
|
+
end
|
|
122
|
+
|
|
115
123
|
def test_files
|
|
116
124
|
world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
|
|
117
125
|
end
|
|
@@ -131,17 +139,21 @@ module Saboteur
|
|
|
131
139
|
failure_details: listener.failure_details)
|
|
132
140
|
end
|
|
133
141
|
|
|
134
|
-
def
|
|
135
|
-
|
|
142
|
+
def example_count(files: nil)
|
|
143
|
+
world.example_count(runnable_groups(files))
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def run_all(recorder, files: nil, seed: nil)
|
|
147
|
+
configuration.order = "rand:#{seed || config.fetch('seed', 1)}"
|
|
136
148
|
configuration.fail_fast = false
|
|
137
|
-
execute(Listener.new(recorder), log_phase: true)
|
|
149
|
+
execute(Listener.new(recorder), log_phase: true, files: files)
|
|
138
150
|
end
|
|
139
151
|
|
|
140
|
-
def run_baseline(seed:)
|
|
152
|
+
def run_baseline(seed:, files: nil)
|
|
141
153
|
configuration.order = "rand:#{seed}"
|
|
142
154
|
configuration.fail_fast = false
|
|
143
155
|
listener = Listener.new
|
|
144
|
-
passed = execute(listener, log_phase: true)
|
|
156
|
+
passed = execute(listener, log_phase: true, files: files)
|
|
145
157
|
RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
|
|
146
158
|
executed_ids: listener.executed, runtime: listener.runtime,
|
|
147
159
|
failure_classes: listener.failure_classes, failure_messages: listener.failure_messages,
|
|
@@ -190,6 +202,8 @@ module Saboteur
|
|
|
190
202
|
'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
|
|
191
203
|
"suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
|
|
192
204
|
'this; see README "Rails engines and dummy apps".'
|
|
205
|
+
hints = EnvironmentHints.all(config.root)
|
|
206
|
+
message << "\nSuspect: #{hints.join('; ')}." if hints.any?
|
|
193
207
|
if coverage_suspect?(captured_text)
|
|
194
208
|
message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
|
|
195
209
|
"Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
|
|
@@ -260,10 +274,10 @@ module Saboteur
|
|
|
260
274
|
# redirected to the phase's log: progress and failure detail land in
|
|
261
275
|
# that log, so a failed phase can be diagnosed from it. Mutant runs
|
|
262
276
|
# stay quiet.
|
|
263
|
-
def execute(listener, log_phase: false)
|
|
277
|
+
def execute(listener, log_phase: false, files: nil)
|
|
264
278
|
reporter = ::RSpec::Core::Reporter.new(configuration)
|
|
265
279
|
reporter.register_listener(listener, *Listener::EVENTS)
|
|
266
|
-
groups =
|
|
280
|
+
groups = runnable_groups(files)
|
|
267
281
|
count = world.example_count(groups)
|
|
268
282
|
if log_phase
|
|
269
283
|
logger = PhaseLogger.new($stdout, total: count)
|
|
@@ -277,6 +291,31 @@ module Saboteur
|
|
|
277
291
|
passed && !world.non_example_failure
|
|
278
292
|
end
|
|
279
293
|
|
|
294
|
+
# The top-level groups with examples to run, in the world's order. With
|
|
295
|
+
# files, only the groups a spec file in that list defines (a part of a
|
|
296
|
+
# partitioned coverage build, #741). Both sides are compared as real
|
|
297
|
+
# paths. An empty list is refused and so is a listed file that defines
|
|
298
|
+
# no group: a part that silently runs nothing would leave its tests
|
|
299
|
+
# out of the map.
|
|
300
|
+
def runnable_groups(files)
|
|
301
|
+
groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
|
|
302
|
+
return groups unless files
|
|
303
|
+
raise ArgumentError, "files must name at least one spec file" if files.empty?
|
|
304
|
+
|
|
305
|
+
wanted = files.to_set { |file| File.realpath(file) }
|
|
306
|
+
chosen = groups.select { |group| wanted.include?(real_path(group.metadata[:file_path])) }
|
|
307
|
+
missing = wanted - chosen.to_set { |group| real_path(group.metadata[:file_path]) }
|
|
308
|
+
raise ArgumentError, "no example group in: #{missing.sort.join(', ')}" unless missing.empty?
|
|
309
|
+
|
|
310
|
+
chosen
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def real_path(path)
|
|
314
|
+
File.realpath(path)
|
|
315
|
+
rescue SystemCallError
|
|
316
|
+
File.expand_path(path)
|
|
317
|
+
end
|
|
318
|
+
|
|
280
319
|
# Progress and failure detail for a phase log. Every 25 finished
|
|
281
320
|
# examples is enough to see movement on a tail; failures print RSpec's
|
|
282
321
|
# full formatted detail, and the run ends with RSpec's totals line.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# Records which lines ran, per file, with a TracePoint on :line (#725). A
|
|
6
|
+
# per-test recorder can drain it between tests instead of calling
|
|
7
|
+
# Coverage.result, whose read walks every file of the process. Pass `files`
|
|
8
|
+
# (canonical paths) to keep only those; nil keeps all. Not wired into the
|
|
9
|
+
# recorder yet: it matches Coverage's lines mode on saboteur's own suite
|
|
10
|
+
# line for line, but has no branches, so the recorder's branch data still
|
|
11
|
+
# needs its own design.
|
|
12
|
+
class LineHits
|
|
13
|
+
def initialize(files = nil, canonical: ->(path) { path })
|
|
14
|
+
@wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
|
|
15
|
+
@hits = new_hits
|
|
16
|
+
@trace = TracePoint.new(:line, &method(:record))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def enable
|
|
20
|
+
@trace.enable
|
|
21
|
+
self
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def disable
|
|
25
|
+
@trace.disable
|
|
26
|
+
self
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Stops recording without touching the TracePoint: toggling it costs
|
|
30
|
+
# milliseconds once an app is loaded, so a recorder pauses around its own
|
|
31
|
+
# work instead.
|
|
32
|
+
def pause
|
|
33
|
+
@paused = true
|
|
34
|
+
self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def resume
|
|
38
|
+
@paused = false
|
|
39
|
+
self
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# file => Set of line numbers since the last drain. A failure inside the
|
|
43
|
+
# tracer is raised here: raising inside the TracePoint would unwind the
|
|
44
|
+
# program being traced.
|
|
45
|
+
def drain(clear: true)
|
|
46
|
+
failure = @failure
|
|
47
|
+
@failure = nil
|
|
48
|
+
raise failure if failure
|
|
49
|
+
|
|
50
|
+
hits = @hits
|
|
51
|
+
@hits = new_hits if clear
|
|
52
|
+
hits
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def new_hits
|
|
58
|
+
Hash.new { |hash, file| hash[file] = Set.new }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A path the canonicalizer refuses is not a wanted file: a NUL byte in an
|
|
62
|
+
# eval'd file name raises ArgumentError, and a name that is not a file
|
|
63
|
+
# ("(eval at ...)") raises ENOENT from a realpath canonicalizer.
|
|
64
|
+
def wanted?(files, canonical, path)
|
|
65
|
+
files.include?(canonical.call(path))
|
|
66
|
+
rescue ArgumentError, SystemCallError, TypeError
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def record(event)
|
|
71
|
+
return if @paused
|
|
72
|
+
|
|
73
|
+
path = event.path
|
|
74
|
+
return if @wanted && !@wanted[path]
|
|
75
|
+
|
|
76
|
+
@hits[path] << event.lineno
|
|
77
|
+
rescue StandardError => e
|
|
78
|
+
@failure ||= e
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# Records which Ruby methods were entered, by [first line, name] per file,
|
|
6
|
+
# with a TracePoint on :call. Coverage's methods report does the same job
|
|
7
|
+
# but its read walks every method in the process (about 8 ms per read once
|
|
8
|
+
# an app is loaded), and the recorder reads twice per example. :call
|
|
9
|
+
# events are rare next to :line events, and an endless def fires one too
|
|
10
|
+
# (#107). Pass `files` (canonical paths) to keep only those; nil keeps all.
|
|
11
|
+
class MethodCalls
|
|
12
|
+
# The tracer the executable arms before the app loads; the engine reads
|
|
13
|
+
# it once after boot and disables it.
|
|
14
|
+
def self.boot
|
|
15
|
+
@boot ||= new.enable
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.booted
|
|
19
|
+
@boot
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(files = nil, canonical: ->(path) { path })
|
|
23
|
+
@wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
|
|
24
|
+
@hits = new_hits
|
|
25
|
+
@trace = TracePoint.new(:call, &method(:record))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def enable
|
|
29
|
+
@trace.enable unless @trace.enabled?
|
|
30
|
+
self
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def disable
|
|
34
|
+
@trace.disable if @trace.enabled?
|
|
35
|
+
self
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Stops recording without touching the TracePoint: toggling it costs
|
|
39
|
+
# milliseconds once an app is loaded, so the recorder pauses per read.
|
|
40
|
+
def pause
|
|
41
|
+
@paused = true
|
|
42
|
+
self
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def resume
|
|
46
|
+
@paused = false
|
|
47
|
+
self
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# file => Set of [line, name] since the last drain. A failure inside the
|
|
51
|
+
# tracer is raised here: raising inside the TracePoint would unwind the
|
|
52
|
+
# program being traced (and kill a forked child outright).
|
|
53
|
+
def drain(clear: true)
|
|
54
|
+
failure = @failure
|
|
55
|
+
@failure = nil
|
|
56
|
+
raise failure if failure
|
|
57
|
+
|
|
58
|
+
hits = @hits
|
|
59
|
+
@hits = new_hits if clear
|
|
60
|
+
hits
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def new_hits
|
|
66
|
+
Hash.new { |hash, file| hash[file] = Set.new }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# A path the canonicalizer refuses (a NUL byte in an eval'd file name
|
|
70
|
+
# raises ArgumentError) is not a wanted file.
|
|
71
|
+
def wanted?(files, canonical, path)
|
|
72
|
+
files.include?(canonical.call(path))
|
|
73
|
+
rescue ArgumentError
|
|
74
|
+
false
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def record(event)
|
|
78
|
+
# self.equal?(...) never sends a message to the traced receiver, which
|
|
79
|
+
# may be a BasicObject proxy that answers every message (ActiveSupport's
|
|
80
|
+
# deprecation proxies do).
|
|
81
|
+
return if @paused || equal?(event.self) || event.event != :call
|
|
82
|
+
|
|
83
|
+
path = event.path
|
|
84
|
+
return if @wanted && !@wanted[path]
|
|
85
|
+
|
|
86
|
+
@hits[path] << [event.lineno, event.method_id]
|
|
87
|
+
rescue StandardError => e
|
|
88
|
+
@failure ||= e
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|