saboteur 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +97 -0
- data/README.md +16 -3
- data/lib/saboteur/baseline/command.rb +6 -1
- data/lib/saboteur/baseline/comparison.rb +8 -0
- data/lib/saboteur/baseline/completeness.rb +14 -4
- data/lib/saboteur/baseline/evidence.rb +8 -1
- data/lib/saboteur/baseline/run_evidence.rb +10 -1
- data/lib/saboteur/baseline/workflow.rb +17 -5
- data/lib/saboteur/cli/color_option.rb +26 -0
- data/lib/saboteur/cli/doctor_command.rb +1 -0
- data/lib/saboteur/cli/exit_status.rb +12 -7
- data/lib/saboteur/cli/explain_command.rb +2 -1
- data/lib/saboteur/cli/failure_output.rb +13 -2
- data/lib/saboteur/cli/help_command.rb +5 -0
- data/lib/saboteur/cli/history_command.rb +2 -0
- data/lib/saboteur/cli/output_options.rb +1 -5
- data/lib/saboteur/cli/run_command.rb +1 -1
- data/lib/saboteur/cli/run_reporting.rb +2 -10
- data/lib/saboteur/cli/run_validation.rb +0 -1
- data/lib/saboteur/cli/selection_options.rb +3 -2
- data/lib/saboteur/cli/survivors_command.rb +81 -24
- data/lib/saboteur/cli/whole_suite_map_lookup.rb +33 -0
- data/lib/saboteur/cli.rb +6 -2
- data/lib/saboteur/engine/coverage_store.rb +7 -4
- data/lib/saboteur/engine/diagnostics.rb +27 -5
- data/lib/saboteur/engine/pipelines.rb +9 -2
- data/lib/saboteur/engine/survivor_confirmation.rb +75 -11
- data/lib/saboteur/engine/test_guess.rb +2 -1
- data/lib/saboteur/engine.rb +37 -18
- data/lib/saboteur/execution/runner.rb +3 -0
- data/lib/saboteur/gates.rb +1 -1
- data/lib/saboteur/help.rb +5 -5
- data/lib/saboteur/integration/rspec.rb +30 -0
- data/lib/saboteur/reporter/default_findings.rb +1 -1
- data/lib/saboteur/reporter/findings.rb +10 -1
- data/lib/saboteur/reporter/gap_kinds.rb +6 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/next_steps.rb +3 -1
- data/lib/saboteur/reporter/score_table.rb +6 -11
- data/lib/saboteur/reporter/stryker.rb +7 -2
- data/lib/saboteur/reporter/survivor_classification.rb +15 -0
- data/lib/saboteur/reporter/survivor_list.rb +6 -6
- data/lib/saboteur/reporter/terminal.rb +6 -3
- data/lib/saboteur/reporter/unconfirmed_section.rb +44 -0
- data/lib/saboteur/result.rb +39 -2
- data/lib/saboteur/selection/coverage_map.rb +11 -1
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +4 -0
- metadata +4 -1
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
|
+
require "stringio"
|
|
5
6
|
|
|
6
7
|
require_relative "../survivor_guidance"
|
|
8
|
+
require_relative "../reporter/palette"
|
|
7
9
|
require_relative "../reporter/survivor_hints"
|
|
8
10
|
|
|
9
11
|
module Saboteur
|
|
@@ -17,6 +19,11 @@ module Saboteur
|
|
|
17
19
|
# Where `saboteur run` writes the JSON report unless configured otherwise.
|
|
18
20
|
DEFAULT_REPORT = Config::DEFAULTS["json_report"]
|
|
19
21
|
|
|
22
|
+
# survivors owns its usage line (#1150): the report path is optional
|
|
23
|
+
# because the run always writes the same default.
|
|
24
|
+
SURVIVORS_USAGE = "Usage: saboteur survivors [options] [REPORT] - REPORT defaults to " \
|
|
25
|
+
".saboteur/report.json"
|
|
26
|
+
|
|
20
27
|
private
|
|
21
28
|
|
|
22
29
|
def survivors_command
|
|
@@ -27,28 +34,53 @@ module Saboteur
|
|
|
27
34
|
end
|
|
28
35
|
|
|
29
36
|
def survivors_command_inner
|
|
30
|
-
parser = OptionParser.new do |opts|
|
|
31
|
-
opts.on("--ranked", "Review-priority order: the report's ranked findings first")
|
|
32
|
-
opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
|
|
33
|
-
opts.on("--operator NAME", "Only survivors of operator NAME")
|
|
34
|
-
opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
|
|
35
|
-
opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
|
|
36
|
-
end
|
|
37
37
|
options = {}
|
|
38
|
-
|
|
38
|
+
parser = survivors_parser(options)
|
|
39
|
+
args = parse_survivors_args(parser, options)
|
|
40
|
+
if options[:help]
|
|
41
|
+
@out.puts(parser)
|
|
42
|
+
return EXIT_OK
|
|
43
|
+
end
|
|
39
44
|
path = args.shift || DEFAULT_REPORT
|
|
40
|
-
raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
|
|
45
|
+
raise UsageError, "unexpected arguments: #{args.join(' ')}\n#{parser}" if args.any?
|
|
41
46
|
|
|
42
47
|
report = load_survivor_report(path)
|
|
43
|
-
browser = SurvivorsBrowser.new(report)
|
|
48
|
+
browser = SurvivorsBrowser.new(report, palette: Reporter::Palette.for(@out, mode: options.fetch(:color, :auto)))
|
|
44
49
|
if options[:show]
|
|
45
50
|
@out.puts(browser.show(options[:show]))
|
|
46
51
|
else
|
|
47
|
-
|
|
52
|
+
# Explicit keywords, and list takes no catch-all (#1267): a new
|
|
53
|
+
# parser option that never reaches the browser raises here instead
|
|
54
|
+
# of being swallowed. --ranked is the documented compat no-op.
|
|
55
|
+
@out.puts(browser.list(file: options[:file], operator: options[:operator], triage: options[:triage]))
|
|
48
56
|
end
|
|
49
57
|
EXIT_OK
|
|
50
58
|
end
|
|
51
59
|
|
|
60
|
+
# The same colour flags as run (#1150), plus the browser filters.
|
|
61
|
+
def survivors_parser(options)
|
|
62
|
+
OptionParser.new do |opts|
|
|
63
|
+
opts.banner = SURVIVORS_USAGE
|
|
64
|
+
opts.on("--ranked", "Review-priority order (the default): the terminal report's ordering")
|
|
65
|
+
opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
|
|
66
|
+
opts.on("--operator NAME", "Only survivors of operator NAME")
|
|
67
|
+
opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
|
|
68
|
+
opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
|
|
69
|
+
opts.on("--color [MODE]", %w[always never auto],
|
|
70
|
+
"Colour the output: always, never, or auto when MODE is omitted") do |mode|
|
|
71
|
+
options[:color] = (mode || :always).to_sym
|
|
72
|
+
end
|
|
73
|
+
opts.on("--no-color", "Alias for --color=never") { options[:color] = :never }
|
|
74
|
+
opts.on("--help", "Show this help") { options[:help] = true }
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def parse_survivors_args(parser, options)
|
|
79
|
+
parser.parse!(@argv, into: options)
|
|
80
|
+
rescue OptionParser::ParseError => e
|
|
81
|
+
raise UsageError, "#{e.message}\n#{parser}"
|
|
82
|
+
end
|
|
83
|
+
|
|
52
84
|
def load_survivor_report(path)
|
|
53
85
|
unless File.file?(path)
|
|
54
86
|
hint = path == DEFAULT_REPORT ? "; run saboteur first, or pass a report path" : ""
|
|
@@ -64,14 +96,21 @@ module Saboteur
|
|
|
64
96
|
# Reads the report's files section (the complete mutant list), not the
|
|
65
97
|
# capped findings, so survivors beyond the findings limit still list.
|
|
66
98
|
class SurvivorsBrowser
|
|
67
|
-
|
|
99
|
+
# Sibling digests named on a collapse line; the rest are a count - the
|
|
100
|
+
# same shape the terminal report's collapse uses (#1150).
|
|
101
|
+
SIBLING_DIGEST_CAP = 3
|
|
102
|
+
|
|
103
|
+
def initialize(report, palette: Reporter::Palette.for(StringIO.new))
|
|
68
104
|
@report = report
|
|
69
105
|
@findings = report.dig("saboteur", "findings") || {}
|
|
106
|
+
@palette = palette
|
|
70
107
|
end
|
|
71
108
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
109
|
+
# One ordering everywhere (#1150): the terminal report's ranked order
|
|
110
|
+
# is the default, exactly as the findings carry it; --ranked stays
|
|
111
|
+
# accepted for compatibility.
|
|
112
|
+
def list(file: nil, operator: nil, triage: nil)
|
|
113
|
+
survivors = ranked_order(filtered(file, operator, triage))
|
|
75
114
|
return "no survivors#{filters_note(file, operator, triage)}" if survivors.empty?
|
|
76
115
|
|
|
77
116
|
[header(survivors.size, file, operator, triage), *rows(survivors)]
|
|
@@ -160,19 +199,32 @@ module Saboteur
|
|
|
160
199
|
def header(size, file, operator, triage)
|
|
161
200
|
text = "#{size} survivor#{'s' unless size == 1}#{filters_note(file, operator, triage)}"
|
|
162
201
|
truncated = @findings["survivorsTruncated"].to_i
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
202
|
+
if truncated.positive?
|
|
203
|
+
text = "#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
|
|
204
|
+
end
|
|
205
|
+
@palette.paint(:head, text)
|
|
166
206
|
end
|
|
167
207
|
|
|
208
|
+
# The terminal collapse: sibling mutants on one statement fold into the
|
|
209
|
+
# first row, which names up to SIBLING_DIGEST_CAP of their digests.
|
|
168
210
|
def rows(survivors)
|
|
169
|
-
survivors.each_with_index.map do |
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
211
|
+
survivors.group_by { |s| [s[:path], s[:line]] }.values.each_with_index.map do |group, i|
|
|
212
|
+
first, *siblings = group
|
|
213
|
+
format("%3<i>d. %<path>s:%<line>d %-28<operator>s %<digest>s %<classification>s%<suffix>s",
|
|
214
|
+
i: i + 1, path: first[:path], line: first[:line], operator: first[:operator],
|
|
215
|
+
digest: first[:digest], classification: first[:classification] || "-",
|
|
216
|
+
suffix: siblings_suffix(siblings))
|
|
173
217
|
end
|
|
174
218
|
end
|
|
175
219
|
|
|
220
|
+
def siblings_suffix(siblings)
|
|
221
|
+
return "" if siblings.empty?
|
|
222
|
+
|
|
223
|
+
named = siblings.first(SIBLING_DIGEST_CAP).map { |s| s[:digest] }.join(", ")
|
|
224
|
+
rest = siblings.size - SIBLING_DIGEST_CAP
|
|
225
|
+
" (+#{siblings.size} here: #{named}#{", +#{rest} more" if rest.positive?})"
|
|
226
|
+
end
|
|
227
|
+
|
|
176
228
|
def ignore_entry(survivor, subject)
|
|
177
229
|
block = if subject
|
|
178
230
|
SurvivorGuidance.ignore_block(survivor[:digest], subject)
|
|
@@ -192,12 +244,17 @@ module Saboteur
|
|
|
192
244
|
subject: m["subject"], covered: m["coveredBy"] || [], source: entry["source"] }
|
|
193
245
|
end
|
|
194
246
|
end
|
|
195
|
-
survivors
|
|
247
|
+
survivors
|
|
196
248
|
end
|
|
197
249
|
end
|
|
198
250
|
|
|
251
|
+
# Ranked order straight from the report: the uncapped findings "order"
|
|
252
|
+
# list first, with the capped findings as the fallback for older
|
|
253
|
+
# reports; anything unknown trails in file order.
|
|
199
254
|
def ranked_order(survivors)
|
|
200
|
-
|
|
255
|
+
order = @findings["order"] ||
|
|
256
|
+
(@findings.fetch("survivors", []) + @findings.fetch("accepted", [])).map { |f| f["digest"] }
|
|
257
|
+
rank = order.each_with_index.to_h
|
|
201
258
|
survivors.sort_by { |s| [rank[s[:digest]] || Float::INFINITY, -s[:covered].size, s[:path], s[:line], s[:digest]] }
|
|
202
259
|
end
|
|
203
260
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The whole-suite coverage map a finished run hands to survivor
|
|
7
|
+
# presentation, and the notice for a scoped run that has none.
|
|
8
|
+
module WholeSuiteMapLookup
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
# No engine or no cache: no whole-suite map can exist, so the survivor
|
|
12
|
+
# block keeps its legacy covered-by line over an impossible build (review C1).
|
|
13
|
+
def whole_suite_map_for(engine, report)
|
|
14
|
+
warn_survivors_unconfirmed(report, engine)
|
|
15
|
+
return engine.whole_suite_coverage_map if engine&.whole_suite_map_lookup?
|
|
16
|
+
|
|
17
|
+
Reporter::SurvivorList::NO_WHOLE_SUITE_LOOKUP
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# A scoped run's survivors may be pinned by a spec it never loaded.
|
|
21
|
+
# With no whole-suite map to check, say so once, even under --quiet
|
|
22
|
+
# (#1286). Verdicts, score and exit code are unchanged.
|
|
23
|
+
def warn_survivors_unconfirmed(report, engine)
|
|
24
|
+
return if report.unignored_survivors.empty?
|
|
25
|
+
return unless engine&.whole_suite_map_lookup? && engine.scoped_selection?
|
|
26
|
+
return if engine.whole_suite_coverage_map
|
|
27
|
+
|
|
28
|
+
@err.puts "saboteur: survivors unconfirmed: no whole-suite coverage map; " \
|
|
29
|
+
"run `saboteur run --all` to build one"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -13,7 +13,9 @@ require_relative "cli/error_cause"
|
|
|
13
13
|
require_relative "cli/exit_status"
|
|
14
14
|
require_relative "cli/baseline_gate"
|
|
15
15
|
require_relative "preview"
|
|
16
|
+
require_relative "cli/color_option"
|
|
16
17
|
require_relative "cli/run_reporting"
|
|
18
|
+
require_relative "cli/whole_suite_map_lookup"
|
|
17
19
|
require_relative "cli/survivors_command"
|
|
18
20
|
require_relative "cli/explain_command"
|
|
19
21
|
require_relative "cli/source_set"
|
|
@@ -43,7 +45,9 @@ module Saboteur
|
|
|
43
45
|
include Baseline::Command
|
|
44
46
|
include ExitStatus
|
|
45
47
|
include BaselineGate
|
|
48
|
+
include ColorOption
|
|
46
49
|
include RunReporting
|
|
50
|
+
include WholeSuiteMapLookup
|
|
47
51
|
include SurvivorsCommand
|
|
48
52
|
include ExplainCommand
|
|
49
53
|
include SourceSet
|
|
@@ -100,7 +104,7 @@ module Saboteur
|
|
|
100
104
|
end
|
|
101
105
|
|
|
102
106
|
def call
|
|
103
|
-
command = @argv.shift
|
|
107
|
+
command = @command = @argv.shift
|
|
104
108
|
case command
|
|
105
109
|
when "run" then run_command
|
|
106
110
|
when "baseline" then baseline
|
|
@@ -109,7 +113,7 @@ module Saboteur
|
|
|
109
113
|
when "doctor" then doctor_command
|
|
110
114
|
when "badge", "trend" then history_entry(command)
|
|
111
115
|
when "version", "--version" then @out.puts(VERSION) || EXIT_OK
|
|
112
|
-
when "help", "--help", "-h" then
|
|
116
|
+
when "help", "--help", "-h" then help_command
|
|
113
117
|
when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
|
|
114
118
|
else preset_or_usage_error(command)
|
|
115
119
|
end
|
|
@@ -18,7 +18,10 @@ module Saboteur
|
|
|
18
18
|
CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
|
|
19
19
|
# Names the freshest whole-suite map build; read as an untrusted hint:
|
|
20
20
|
# only a bare 64-hex key joins the cache dir (#427).
|
|
21
|
-
|
|
21
|
+
# Renamed from "whole-suite-map" in 0.12.2: older versions also wrote
|
|
22
|
+
# the pointer for maps built from inferred specs, which are not whole
|
|
23
|
+
# suites, so their pointers are never read (#1286).
|
|
24
|
+
WHOLE_SUITE_POINTER = "whole-suite-map-v2"
|
|
22
25
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
23
26
|
LOG = "coverage-build.log"
|
|
24
27
|
# RSpec rewrites this on every run (example_status_persistence_file_path),
|
|
@@ -65,7 +68,7 @@ module Saboteur
|
|
|
65
68
|
return serve_cached(path, cached, files, preloaded) if cached
|
|
66
69
|
|
|
67
70
|
note = miss_note(files)
|
|
68
|
-
@diagnostics.
|
|
71
|
+
@diagnostics.notice(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
|
|
69
72
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
70
73
|
map = fresh_map(files, note, preloaded)
|
|
71
74
|
build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
@@ -237,7 +240,7 @@ module Saboteur
|
|
|
237
240
|
return build_parts(files, parts) if parts.size > 1
|
|
238
241
|
|
|
239
242
|
scope = @config["tests"].empty? ? "the whole suite" : "the selected specs"
|
|
240
|
-
@diagnostics.
|
|
243
|
+
@diagnostics.notice("building coverage map (runs #{scope} once)")
|
|
241
244
|
# The suite child forked next inherits this heap; pack it first, to
|
|
242
245
|
# reduce later copying of shared pages.
|
|
243
246
|
Execution::ForkReadiness.prepare_heap
|
|
@@ -267,7 +270,7 @@ module Saboteur
|
|
|
267
270
|
total, counts = usage_checked { check_parts_cover_suite(parts) }
|
|
268
271
|
saved = saved_parts(files, parts.size, counts)
|
|
269
272
|
todo = saved.each_index.select { |index| saved[index].nil? }
|
|
270
|
-
@diagnostics.
|
|
273
|
+
@diagnostics.notice(parts_note(parts.size, todo.size))
|
|
271
274
|
built = todo.empty? ? [] : run_parts(files, parts, todo)
|
|
272
275
|
todo.zip(built) { |index, map| saved[index] = map }
|
|
273
276
|
map = combine_saved(saved, total, files, parts.size)
|
|
@@ -17,6 +17,12 @@ module Saboteur
|
|
|
17
17
|
# be watched: the heartbeat then adds the count, the percentage and an
|
|
18
18
|
# estimate of the time left to its elapsed-time line.
|
|
19
19
|
class ProgressHeartbeat
|
|
20
|
+
# --quiet substitute: responds to the supervisor's calls, prints nothing.
|
|
21
|
+
NULL = Object.new.tap do |noop|
|
|
22
|
+
def noop.watch(*) = self
|
|
23
|
+
def noop.tick = nil
|
|
24
|
+
end
|
|
25
|
+
|
|
20
26
|
COUNT = %r{(\d+)/([1-9]\d*) examples finished}
|
|
21
27
|
TAIL_BYTES = 4096
|
|
22
28
|
|
|
@@ -91,9 +97,10 @@ module Saboteur
|
|
|
91
97
|
attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
92
98
|
:hazard_suppressions, :inline_suppressions, :verbose, :quiet_start
|
|
93
99
|
|
|
94
|
-
def initialize(err, verbose: false, palette: nil, quiet_start: true)
|
|
100
|
+
def initialize(err, verbose: false, palette: nil, quiet_start: true, quiet: false)
|
|
95
101
|
@err = err
|
|
96
102
|
@verbose = verbose
|
|
103
|
+
@quiet = quiet
|
|
97
104
|
@quiet_start = quiet_start
|
|
98
105
|
@palette = palette || Reporter::Palette.for(err)
|
|
99
106
|
@warnings = []
|
|
@@ -180,10 +187,17 @@ module Saboteur
|
|
|
180
187
|
@err.puts("saboteur: #{Text.display_safe_lines(message)}")
|
|
181
188
|
end
|
|
182
189
|
|
|
190
|
+
# First-run explanations (#1264): the --quiet contract is findings,
|
|
191
|
+
# score and verdict, so these stay off stderr under it and print
|
|
192
|
+
# exactly as before otherwise.
|
|
193
|
+
def notice(message)
|
|
194
|
+
inform(message) unless @quiet
|
|
195
|
+
end
|
|
196
|
+
|
|
183
197
|
# Chatter the default view folds into the quiet-start lines;
|
|
184
198
|
# --verbose prints it byte-for-byte as before.
|
|
185
199
|
def detail(message)
|
|
186
|
-
inform(message) if @verbose || !@quiet_start
|
|
200
|
+
inform(message) if !@quiet && (@verbose || !@quiet_start)
|
|
187
201
|
end
|
|
188
202
|
|
|
189
203
|
# A selection explanation the default view carries as one dim
|
|
@@ -199,7 +213,7 @@ module Saboteur
|
|
|
199
213
|
# The quiet start: three lines, then progress. --verbose keeps
|
|
200
214
|
# today's chatter instead and never prints these.
|
|
201
215
|
def quiet_start_header(scope:, seed:, operators:)
|
|
202
|
-
return if @verbose || !@quiet_start
|
|
216
|
+
return if @quiet || @verbose || !@quiet_start
|
|
203
217
|
|
|
204
218
|
dot = @palette.glyph(:middot)
|
|
205
219
|
line = "saboteur #{Saboteur::VERSION} #{dot} #{scope} #{dot} seed #{seed} #{dot} " \
|
|
@@ -208,7 +222,7 @@ module Saboteur
|
|
|
208
222
|
end
|
|
209
223
|
|
|
210
224
|
def quiet_baseline(passes:, elapsed:, tests:, cached:)
|
|
211
|
-
return if @verbose || !@quiet_start
|
|
225
|
+
return if @quiet || @verbose || !@quiet_start
|
|
212
226
|
|
|
213
227
|
check = @palette.paint(:pass, @palette.glyph(:check))
|
|
214
228
|
suffix = cached ? " (cached)" : ""
|
|
@@ -219,7 +233,7 @@ module Saboteur
|
|
|
219
233
|
end
|
|
220
234
|
|
|
221
235
|
def quiet_inference(spec_count)
|
|
222
|
-
return if spec_count.nil? || @verbose || !@quiet_start
|
|
236
|
+
return if spec_count.nil? || @quiet || @verbose || !@quiet_start
|
|
223
237
|
|
|
224
238
|
files = spec_count == 1 ? "spec file" : "spec files"
|
|
225
239
|
@err.puts(@palette.paint(:dim, "spec inference: #{spec_count} #{files} (details: --verbose)"))
|
|
@@ -252,6 +266,10 @@ module Saboteur
|
|
|
252
266
|
# fork time. Prints "<label>: N s elapsed" about every interval
|
|
253
267
|
# seconds, so a long phase does not look stuck.
|
|
254
268
|
def progress_heartbeat(label, interval: 30)
|
|
269
|
+
# --quiet: the contract is findings, score and verdict - no elapsed
|
|
270
|
+
# chatter; the heartbeat's ticks become no-ops.
|
|
271
|
+
return ProgressHeartbeat::NULL if @quiet
|
|
272
|
+
|
|
255
273
|
ProgressHeartbeat.new(self, label, interval: interval)
|
|
256
274
|
end
|
|
257
275
|
|
|
@@ -261,6 +279,8 @@ module Saboteur
|
|
|
261
279
|
# The count lives in one file the workers inherit, so every worker adds
|
|
262
280
|
# to the same number.
|
|
263
281
|
def start_progress(total)
|
|
282
|
+
return if @quiet
|
|
283
|
+
|
|
264
284
|
@progress_total = total
|
|
265
285
|
@progress_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
266
286
|
@progress_file = Tempfile.new("saboteur-progress")
|
|
@@ -271,6 +291,8 @@ module Saboteur
|
|
|
271
291
|
end
|
|
272
292
|
|
|
273
293
|
def progress(result)
|
|
294
|
+
return if @quiet
|
|
295
|
+
|
|
274
296
|
mark = MARKS.fetch(result.status, "?")
|
|
275
297
|
return plain_progress(painted_mark(mark)) unless @progress_file
|
|
276
298
|
|
|
@@ -104,9 +104,16 @@ module Saboteur
|
|
|
104
104
|
# "Full run" for ignore staleness (#1206): no --since diff, no --subject
|
|
105
105
|
# filter, and no narrowed include (explicit file arguments replace the
|
|
106
106
|
# configured include) - only then was every in-scope subject examined, so
|
|
107
|
-
# an entry whose subject never appears really points at gone code.
|
|
107
|
+
# an entry whose subject never appears really points at gone code. The
|
|
108
|
+
# include nobody set counts as unscoped whether it is the generic
|
|
109
|
+
# default or the Rails one a config-less app gets (#1255).
|
|
108
110
|
def unscoped_full_run?
|
|
109
|
-
!config["since"] && config["subjects"].empty? &&
|
|
111
|
+
!config["since"] && config["subjects"].empty? && default_include?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def default_include?
|
|
115
|
+
config["include"] == Config::DEFAULTS["include"] ||
|
|
116
|
+
(config["include"] == Config::RAILS_INCLUDE && !config.explicit?("include"))
|
|
110
117
|
end
|
|
111
118
|
|
|
112
119
|
def external_command? = !config["test_command"].to_s.empty?
|
|
@@ -15,13 +15,14 @@ module Saboteur
|
|
|
15
15
|
# confirmation_budget caps the whole pass.
|
|
16
16
|
MAX_SPECS_PER_SURVIVOR = 8
|
|
17
17
|
|
|
18
|
-
def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil)
|
|
18
|
+
def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil, load_specs: ->(_files) {})
|
|
19
19
|
@config = config
|
|
20
20
|
@map = map
|
|
21
21
|
@scheduler = scheduler
|
|
22
22
|
@diagnostics = diagnostics
|
|
23
23
|
@budget_exceeded = budget_exceeded
|
|
24
24
|
@cache = cache
|
|
25
|
+
@load_specs = load_specs
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
def call(results, ignore_list:, early_stop: nil)
|
|
@@ -33,19 +34,60 @@ module Saboteur
|
|
|
33
34
|
"stopped early (#{early_stop.reason})")
|
|
34
35
|
end
|
|
35
36
|
@remaining = @config["confirmation_budget"]
|
|
37
|
+
@map_silent = 0
|
|
36
38
|
results.map { |result| survivors.include?(result) ? confirm(result) : result }
|
|
37
39
|
.tap { |confirmed| summarize(confirmed) }
|
|
40
|
+
.tap { warn_map_silent }
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
private
|
|
41
44
|
|
|
45
|
+
# A crash while confirming one survivor downgrades it to unconfirmed
|
|
46
|
+
# with one warning; it never ends the run (#1140).
|
|
42
47
|
def confirm(result)
|
|
48
|
+
confirm_checked(result)
|
|
49
|
+
rescue RefusedSpecPath => e
|
|
50
|
+
warn_refused(e)
|
|
51
|
+
result.with_confirmation("unconfirmed_map(a covering spec from the whole-suite map was refused)")
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
warn_failed(result, e)
|
|
54
|
+
unconfirmed(result, "confirmation failed: #{e.class}")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The map named a spec that is not a spec file inside the project: the
|
|
58
|
+
# map cannot be trusted, so its survivors stay counted (#1286).
|
|
59
|
+
def warn_refused(error)
|
|
60
|
+
return if @warned_refused
|
|
61
|
+
|
|
62
|
+
@warned_refused = true
|
|
63
|
+
@diagnostics.warn("survivor confirmation refused the whole-suite coverage map: " \
|
|
64
|
+
"#{error.message.lines.first.strip}; its survivors stay counted; " \
|
|
65
|
+
"run `saboteur run --all` to rebuild it")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# One line per run, however many survivors hit the same failure.
|
|
69
|
+
def warn_failed(result, error)
|
|
70
|
+
return if @warned_failure
|
|
71
|
+
|
|
72
|
+
@warned_failure = true
|
|
73
|
+
@diagnostics.warn("survivor confirmation failed for #{result.subject.id} #{result.digest} " \
|
|
74
|
+
"(#{error.class}: #{error.message.lines.map(&:strip).reject(&:empty?).first}); " \
|
|
75
|
+
"its survivors stay unconfirmed, outside the score")
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def confirm_checked(result)
|
|
43
79
|
return unconfirmed(result, "the confirmation budget is spent") if @remaining <= 0
|
|
44
80
|
return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
|
|
45
81
|
|
|
46
|
-
|
|
47
|
-
|
|
82
|
+
# Nothing in the whole-suite map covers the edited lines (the tree
|
|
83
|
+
# drifted since the map was built): the map cannot confirm anything.
|
|
84
|
+
covering = covering_tests(result)
|
|
85
|
+
return map_silent(result) if covering.empty?
|
|
48
86
|
|
|
87
|
+
tests = unrun_covering_tests(result, covering)
|
|
88
|
+
return result.with_confirmation("confirmed") if tests.empty?
|
|
89
|
+
|
|
90
|
+
load_unselected_specs(tests)
|
|
49
91
|
outcome = run_confirmation(result, tests)
|
|
50
92
|
@remaining -= outcome.runtime || 0
|
|
51
93
|
case outcome.status
|
|
@@ -58,8 +100,14 @@ module Saboteur
|
|
|
58
100
|
# The whole-suite map's covering tests for the mutant's edited lines
|
|
59
101
|
# (branch evidence intersected, mirroring the scheduler), minus the
|
|
60
102
|
# tests the mutant already ran, fastest first, capped.
|
|
61
|
-
def unrun_covering_tests(result)
|
|
62
|
-
fastest_first(
|
|
103
|
+
def unrun_covering_tests(result, covering)
|
|
104
|
+
fastest_first(covering - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# The covering specs may live in files this run never loaded; the
|
|
108
|
+
# forked confirmation child can only run examples already in its world.
|
|
109
|
+
def load_unselected_specs(tests)
|
|
110
|
+
@load_specs.call(tests.map { |id| id.sub(/\[[\d:]+\]\z/, "") })
|
|
63
111
|
end
|
|
64
112
|
|
|
65
113
|
def covering_tests(result)
|
|
@@ -86,11 +134,12 @@ module Saboteur
|
|
|
86
134
|
# becomes an error; a check the budgets never let run comes back
|
|
87
135
|
# unverified - outside the score and never cached.
|
|
88
136
|
def checked_kill(result, tests, outcome)
|
|
89
|
-
kill = result.with_confirmation(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
137
|
+
kill = result.with_confirmation(
|
|
138
|
+
"killed_by_confirmation", status: :killed, reason: outcome.message, killed_by: outcome.failed_ids,
|
|
139
|
+
covered_by: tests, kill_cause: outcome.kill_cause,
|
|
140
|
+
runtime: (result.runtime || 0) + (outcome.runtime || 0),
|
|
141
|
+
executed: (result.executed || []) + outcome.executed_ids
|
|
142
|
+
)
|
|
94
143
|
checked = VerdictCheck.new(run_unchanged: unchanged_runner(result.subject),
|
|
95
144
|
baseline_timeout: @config["baseline_timeout"]).call([kill]).first
|
|
96
145
|
return checked unless checked.status == :killed
|
|
@@ -108,7 +157,7 @@ module Saboteur
|
|
|
108
157
|
# (#1079). ResultCache#record replaces the subject+mutation entry, so
|
|
109
158
|
# the stale survivor verdict cannot win a later lookup.
|
|
110
159
|
def record_kill(result, kill)
|
|
111
|
-
@cache&.record(kill.with_confirmation("
|
|
160
|
+
@cache&.record(kill.with_confirmation("killed_by_confirmation", covered_by: covering_tests(result)))
|
|
112
161
|
end
|
|
113
162
|
|
|
114
163
|
# The unchanged-method re-run VerdictCheck checks a kill against, in
|
|
@@ -129,6 +178,13 @@ module Saboteur
|
|
|
129
178
|
end
|
|
130
179
|
end
|
|
131
180
|
|
|
181
|
+
# The map says nothing about the edited lines (the tree drifted since it
|
|
182
|
+
# was built): the survivor stays counted and the pass says so once.
|
|
183
|
+
def map_silent(result)
|
|
184
|
+
@map_silent += 1
|
|
185
|
+
result.with_confirmation("unconfirmed_map(no covering tests in the whole-suite map)")
|
|
186
|
+
end
|
|
187
|
+
|
|
132
188
|
def unconfirmed(result, reason)
|
|
133
189
|
result.with_confirmation("unconfirmed(#{reason})")
|
|
134
190
|
end
|
|
@@ -148,6 +204,14 @@ module Saboteur
|
|
|
148
204
|
durations.empty? ? nil : durations.sum / durations.size
|
|
149
205
|
end
|
|
150
206
|
|
|
207
|
+
def warn_map_silent
|
|
208
|
+
return if @map_silent.zero?
|
|
209
|
+
|
|
210
|
+
@diagnostics.warn("survivor confirmation: the whole-suite coverage map covers none of the edited lines of " \
|
|
211
|
+
"#{@map_silent} survivor#{'s' unless @map_silent == 1}, so they stay counted; " \
|
|
212
|
+
"run `saboteur run --all` to rebuild it")
|
|
213
|
+
end
|
|
214
|
+
|
|
151
215
|
def summarize(results)
|
|
152
216
|
counts = results.filter_map(&:confirmation).tally
|
|
153
217
|
return if counts.empty?
|
|
@@ -116,7 +116,8 @@ module Saboteur
|
|
|
116
116
|
# Why the subjectless files needed no spec.
|
|
117
117
|
def subjectless_message(since:)
|
|
118
118
|
detail = since ? "no changed mutation subjects" : "no mutation subjects"
|
|
119
|
-
|
|
119
|
+
who = subjectless.size == 1 ? "it has" : "they have"
|
|
120
|
+
"inferred specs: no spec is required for #{names(subjectless)}: #{who} #{detail}"
|
|
120
121
|
end
|
|
121
122
|
|
|
122
123
|
# The per-file warning when the run proceeds with the mapped
|