saboteur 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +2 -2
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +18 -5
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +3 -1
- data/lib/saboteur/baseline/workflow.rb +57 -9
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/exit_status.rb +10 -5
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +2 -2
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_command.rb +17 -1
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +13 -2
- data/lib/saboteur/cli/survivors_command.rb +82 -18
- data/lib/saboteur/cli.rb +3 -0
- data/lib/saboteur/config.rb +7 -2
- data/lib/saboteur/crash_report.rb +3 -2
- data/lib/saboteur/doctor.rb +11 -2
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/scheduler.rb +23 -21
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +11 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +1 -1
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +57 -9
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +13 -1
- data/lib/saboteur/ignore_list.rb +36 -3
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +29 -7
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/baseline_tail.rb +9 -7
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +160 -10
- data/lib/saboteur/reporter/next_steps.rb +9 -7
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +17 -12
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/stryker.rb +11 -1
- data/lib/saboteur/reporter/survivor_classification.rb +26 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -2
- data/lib/saboteur/reporter/survivor_list.rb +58 -4
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +32 -33
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +39 -8
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +10 -2
data/lib/saboteur/preview.rb
CHANGED
|
@@ -59,8 +59,13 @@ module Saboteur
|
|
|
59
59
|
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
# The default set is what --list-operators prints, so a preview leads
|
|
63
|
+
# with it only when the selection was changed or --verbose asks (#1178).
|
|
62
64
|
def announce_operators
|
|
63
|
-
|
|
65
|
+
return if @json
|
|
66
|
+
return unless @config["verbose"] || @config["operators"].any? || @config["exclude_operators"].any?
|
|
67
|
+
|
|
68
|
+
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
|
|
64
69
|
end
|
|
65
70
|
|
|
66
71
|
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
@@ -22,6 +22,11 @@ module Saboteur
|
|
|
22
22
|
baseline_passes: "--baseline-passes", max_duration: "--max-duration",
|
|
23
23
|
shard: "--shard" }.freeze
|
|
24
24
|
|
|
25
|
+
# Not part of the selection a replay reproduces: a shard is a slice of
|
|
26
|
+
# it, and a report path would overwrite the report the replay came
|
|
27
|
+
# from (the audit preset names whole-project reports, #1185).
|
|
28
|
+
REPLAY_SKIPPED = %i[shard json_report html_report].freeze
|
|
29
|
+
|
|
25
30
|
# CLI-only boolean switches recorded in settings, mapped by value, so
|
|
26
31
|
# an explicit false (--no-max-duration-ok over a config true) survives
|
|
27
32
|
# into the generated commands instead of reverting to the config.
|
|
@@ -40,14 +45,30 @@ module Saboteur
|
|
|
40
45
|
|
|
41
46
|
# Replay one survivor by digest in the same selection. capped: false
|
|
42
47
|
# skips the protocol byte cap, for callers that print to a terminal.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
48
|
+
# report: narrows the replay to the mutant's covering spec files
|
|
49
|
+
# instead of the run's full inferred list (#1148).
|
|
50
|
+
def replay(provenance, digest, capped: true, report: nil)
|
|
51
|
+
words = ["saboteur", "run", "--id", digest,
|
|
52
|
+
*selection(provenance, replay: true, tests: report && covering_files(report, digest)),
|
|
53
|
+
"--seed", provenance.seed.to_s]
|
|
46
54
|
finish(words, provenance, capped: capped)
|
|
47
55
|
end
|
|
48
56
|
|
|
49
57
|
private
|
|
50
58
|
|
|
59
|
+
# A replay needs only the specs that cover the mutant, not every
|
|
60
|
+
# inferred file (#1148): covered_by holds RSpec ids like
|
|
61
|
+
# "./spec/fee_spec.rb[1:2]", so strip the example suffix. nil when
|
|
62
|
+
# the survivor has no covering ids - then the whole selection
|
|
63
|
+
# stays, since dropping it would widen the replay to inference.
|
|
64
|
+
def covering_files(report, digest)
|
|
65
|
+
result = report.results.find { |entry| entry.digest == digest }
|
|
66
|
+
return unless result
|
|
67
|
+
|
|
68
|
+
files = result.covered_by.filter_map { |id| id[%r{\A\.?/?(.+?\.rb)\[}, 1] }.uniq
|
|
69
|
+
files unless files.empty?
|
|
70
|
+
end
|
|
71
|
+
|
|
51
72
|
def finish(words, provenance, capped: true)
|
|
52
73
|
words += ["--", *provenance.sources] unless provenance.sources.empty?
|
|
53
74
|
command = Shellwords.join(words)
|
|
@@ -76,7 +97,7 @@ module Saboteur
|
|
|
76
97
|
# (#317). A typed empty list stays: it may clear a config value.
|
|
77
98
|
def setting_words(provenance, replay: false)
|
|
78
99
|
SETTING_FLAGS.flat_map do |key, flag|
|
|
79
|
-
next [] if replay && key
|
|
100
|
+
next [] if replay && REPLAY_SKIPPED.include?(key)
|
|
80
101
|
next [] unless provenance.settings.key?(key)
|
|
81
102
|
|
|
82
103
|
value = provenance.settings[key]
|
|
@@ -88,13 +109,14 @@ module Saboteur
|
|
|
88
109
|
|
|
89
110
|
# Whole-suite scope is a bare --all, only when no tests resolved;
|
|
90
111
|
# --all never combines with --test.
|
|
91
|
-
def selection(provenance, replay: false)
|
|
112
|
+
def selection(provenance, replay: false, tests: nil)
|
|
113
|
+
tests ||= provenance.tests
|
|
92
114
|
words = []
|
|
93
115
|
words.push("--config", provenance.config_path) if provenance.config_path
|
|
94
116
|
words.push("--since", provenance.ref) if provenance.ref
|
|
95
117
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
96
|
-
words << "--all" if provenance.settings[:all] &&
|
|
97
|
-
words.push(*test_words(
|
|
118
|
+
words << "--all" if provenance.settings[:all] && tests.empty?
|
|
119
|
+
words.push(*test_words(tests))
|
|
98
120
|
words.concat(setting_words(provenance, replay: replay))
|
|
99
121
|
words.concat(mode_words(provenance))
|
|
100
122
|
words.push("--id", provenance.digest) if provenance.digest && !replay
|
|
@@ -40,8 +40,7 @@ module Saboteur
|
|
|
40
40
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
41
41
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
42
42
|
add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
|
|
43
|
-
|
|
44
|
-
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
43
|
+
unverified_reasons(reasons, report)
|
|
45
44
|
discovery_failures(reasons, report)
|
|
46
45
|
early_stop_reason(reasons, report)
|
|
47
46
|
inline_scope(reasons, report)
|
|
@@ -70,6 +69,14 @@ module Saboteur
|
|
|
70
69
|
"included untracked files could not be read; their subjects were not covered")
|
|
71
70
|
end
|
|
72
71
|
|
|
72
|
+
def unverified_reasons(reasons, report)
|
|
73
|
+
flaky = report.results.count(&:flaky?)
|
|
74
|
+
add(reasons, "unverified", report.counts[:unverified] - flaky,
|
|
75
|
+
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
76
|
+
add(reasons, "flaky", flaky,
|
|
77
|
+
"mutants gave a different verdict on the --flaky-check reordered rerun; they are unverified")
|
|
78
|
+
end
|
|
79
|
+
|
|
73
80
|
def early_stop_reason(reasons, report)
|
|
74
81
|
stop = report.early_stop
|
|
75
82
|
return unless stop
|
|
@@ -123,7 +123,8 @@ module Saboteur
|
|
|
123
123
|
steps = {}
|
|
124
124
|
add_command(steps, :rerun, Commands.rerun(provenance))
|
|
125
125
|
first = Findings.build(report)[:survivors].first
|
|
126
|
-
|
|
126
|
+
# report: narrows the replay to covering specs, as on the run screen (#1148).
|
|
127
|
+
add_command(steps, :replay, Commands.replay(provenance, first[:digest], report: report)) if first
|
|
127
128
|
steps[:fullReport] = provenance.json_report if short_path?(provenance.json_report)
|
|
128
129
|
steps
|
|
129
130
|
end
|
|
@@ -24,12 +24,14 @@ module Saboteur
|
|
|
24
24
|
new_count = gate.new_survivor_ids.size
|
|
25
25
|
return pass_line(gate) if new_count.zero?
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
matched = @palette.paint(:dim, "#{matched_count(gate)} matched the baseline#{breakdown(gate)}")
|
|
28
|
+
"saboteur: #{matched} #{@palette.glyph(:middot)} " \
|
|
29
|
+
"#{@palette.paint(:fail, counted(new_count, 'new survivor'))}"
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
def pass_line(gate)
|
|
32
|
-
|
|
33
|
+
matched = @palette.paint(:dim, "#{matched_count(gate)} matched the baseline")
|
|
34
|
+
line = "saboteur: #{matched}, #{@palette.paint(:pass, 'no new survivors')}"
|
|
33
35
|
current, baseline = score_pair(gate)
|
|
34
36
|
line << " - score #{current} (baseline #{baseline})" if current
|
|
35
37
|
line
|
|
@@ -80,13 +82,13 @@ module Saboteur
|
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
def pass_hint
|
|
83
|
-
|
|
84
|
-
|
|
85
|
+
hint = "re-baseline deliberately #{@palette.glyph(:middot)} run without --baseline weekly to catch drift"
|
|
86
|
+
@palette.paint(:info, hint)
|
|
85
87
|
end
|
|
86
88
|
|
|
87
89
|
def fail_hint
|
|
88
|
-
"
|
|
89
|
-
|
|
90
|
+
hint = "re-baseline deliberately: saboteur baseline accept #{@palette.glyph(:middot)} run full weekly"
|
|
91
|
+
" #{@palette.paint(:info, hint)}"
|
|
90
92
|
end
|
|
91
93
|
end
|
|
92
94
|
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# The gap-kind vocabulary (D2): which sort of check a survivor suggests
|
|
7
|
+
# is missing, as a LABEL ONLY (#192) - a heuristic from the operator and
|
|
8
|
+
# the edit shape, never a diagnosis, and never a score input.
|
|
9
|
+
module GapKinds
|
|
10
|
+
KINDS = %w[boundary branch effect number] + ["error path", "result unchecked"].freeze
|
|
11
|
+
|
|
12
|
+
# Operator -> gap kind, every built-in operator exactly once except
|
|
13
|
+
# operand_replacement, whose kind depends on the replaced expression
|
|
14
|
+
# (a comparison loses its boundary; anything else is a plain result).
|
|
15
|
+
# Anything missing here - a new operator before its row lands - is
|
|
16
|
+
# the honest fallback: result unchecked.
|
|
17
|
+
BY_OPERATOR = {
|
|
18
|
+
"comparison" => "boundary",
|
|
19
|
+
"range" => "boundary",
|
|
20
|
+
"endpoint_replacement" => "boundary",
|
|
21
|
+
"condition_negation" => "branch",
|
|
22
|
+
"condition_true" => "branch",
|
|
23
|
+
"condition_false" => "branch",
|
|
24
|
+
"boolean_logic" => "branch",
|
|
25
|
+
"negation_removal" => "branch",
|
|
26
|
+
"when_removal" => "branch",
|
|
27
|
+
"safe_navigation" => "branch",
|
|
28
|
+
"filter_replacement" => "branch",
|
|
29
|
+
"statement_removal" => "effect",
|
|
30
|
+
"element_removal" => "effect",
|
|
31
|
+
"argument_removal" => "effect",
|
|
32
|
+
"pair_removal" => "effect",
|
|
33
|
+
"chain_removal" => "effect",
|
|
34
|
+
"copy_replacement" => "effect",
|
|
35
|
+
"literal_number" => "number",
|
|
36
|
+
"arithmetic" => "number",
|
|
37
|
+
"unary_removal" => "number",
|
|
38
|
+
"operator_assignment" => "number",
|
|
39
|
+
"rescue_removal" => "error path",
|
|
40
|
+
"ensure_removal" => "error path",
|
|
41
|
+
"body_raise" => "error path",
|
|
42
|
+
"body_nil" => "result unchecked",
|
|
43
|
+
"body_true" => "result unchecked",
|
|
44
|
+
"body_false" => "result unchecked",
|
|
45
|
+
"literal_string" => "result unchecked",
|
|
46
|
+
"literal_boolean" => "result unchecked",
|
|
47
|
+
"or_assign" => "result unchecked",
|
|
48
|
+
"and_assign" => "result unchecked",
|
|
49
|
+
"argument_swap" => "result unchecked",
|
|
50
|
+
"method_swap" => "result unchecked",
|
|
51
|
+
"interpolation_removal" => "result unchecked",
|
|
52
|
+
"jump_value" => "result unchecked",
|
|
53
|
+
"regex_replacement" => "result unchecked"
|
|
54
|
+
}.freeze
|
|
55
|
+
|
|
56
|
+
# The one operator whose kind comes from the edit shape, not the name.
|
|
57
|
+
SHAPE_DECIDED = %w[operand_replacement].freeze
|
|
58
|
+
|
|
59
|
+
FALLBACK = "result unchecked"
|
|
60
|
+
|
|
61
|
+
# The deterministic per-kind reason printed next to a survivor's rank.
|
|
62
|
+
REASONS = {
|
|
63
|
+
"boundary" => "the covering specs pass without pinning this boundary",
|
|
64
|
+
"branch" => "the covering specs pass without pinning this branch",
|
|
65
|
+
"effect" => "the covering specs pass without pinning this effect",
|
|
66
|
+
"number" => "the covering specs pass without pinning this number",
|
|
67
|
+
"error path" => "the covering specs pass without pinning this error path",
|
|
68
|
+
"result unchecked" => "the covering specs pass without pinning this result"
|
|
69
|
+
}.freeze
|
|
70
|
+
|
|
71
|
+
module_function
|
|
72
|
+
|
|
73
|
+
def kind(operator, comparison_edit: false)
|
|
74
|
+
return comparison_edit ? "boundary" : FALLBACK if SHAPE_DECIDED.include?(operator)
|
|
75
|
+
|
|
76
|
+
BY_OPERATOR.fetch(operator, FALLBACK)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
require "cgi"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
|
|
7
|
+
require_relative "../survivor_guidance"
|
|
7
8
|
require_relative "survivor_classification"
|
|
9
|
+
require_relative "wording"
|
|
8
10
|
|
|
9
11
|
module Saboteur
|
|
10
12
|
module Reporter
|
|
@@ -31,13 +33,61 @@ module Saboteur
|
|
|
31
33
|
.Accepted,.Triage_accepted_equivalent{background:#8d6e63}
|
|
32
34
|
.Triage_wont_fix{background:#616161}.Triage_todo{background:#b45309}
|
|
33
35
|
.classification{border-radius:3px;padding:0 .4rem;background:#eceff1;color:#37474f;white-space:nowrap}
|
|
36
|
+
.copy{font-size:11px;padding:0 .3rem;margin-left:.3rem;cursor:pointer}
|
|
34
37
|
.warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
|
|
35
38
|
code,pre{font-family:ui-monospace,monospace;font-size:13px}
|
|
36
39
|
pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
|
|
40
|
+
/* Paper cannot scroll sideways, and the dark viewer is screen-only
|
|
41
|
+
ink: print the table, not the widget, and let wide tables wrap. */
|
|
42
|
+
@media print{
|
|
43
|
+
mutation-test-report-app{display:none}
|
|
44
|
+
.copy,.print-hide{display:none}
|
|
45
|
+
details::details-content{content-visibility:visible}
|
|
46
|
+
body{max-width:none}
|
|
47
|
+
table{min-width:0}
|
|
48
|
+
.scroll{overflow-x:visible}
|
|
49
|
+
}
|
|
37
50
|
CSS
|
|
38
51
|
|
|
39
52
|
SCORE_FORMULA = "(killed + timeout) / (killed + survived + timeout)"
|
|
40
53
|
|
|
54
|
+
# A file section holding a survivor, timeout or error opens on load -
|
|
55
|
+
# but only the first OPEN_FILE_LIMIT of them, and only while the open
|
|
56
|
+
# rows stay under OPEN_ROW_BUDGET, so a big run does not land as a wall
|
|
57
|
+
# of expanded rows. The rest stay collapsed, survivor count in the header.
|
|
58
|
+
OPEN_FILE_LIMIT = 10
|
|
59
|
+
OPEN_ROW_BUDGET = 30
|
|
60
|
+
|
|
61
|
+
# Statuses worth opening a file for on load: a survivor, a timeout or an
|
|
62
|
+
# error is exactly what a reader needs to inspect.
|
|
63
|
+
ATTENTION_STATUSES = %w[Survived Timeout RuntimeError].freeze
|
|
64
|
+
|
|
65
|
+
# Survivor rows get a copy button for the replay command. file:// is a
|
|
66
|
+
# secure context, so navigator.clipboard works; the textarea fallback
|
|
67
|
+
# covers browsers where it rejects without a gesture. One delegated
|
|
68
|
+
# listener keeps the page free of per-row scripts.
|
|
69
|
+
COPY_SCRIPT = <<~JS
|
|
70
|
+
document.addEventListener("click",function(event){
|
|
71
|
+
var button=event.target.closest("button.copy");if(!button)return;
|
|
72
|
+
var text=button.getAttribute("data-copy");
|
|
73
|
+
function done(){button.textContent="copied";setTimeout(function(){button.textContent="copy"},1200)}
|
|
74
|
+
function fallback(){var area=document.createElement("textarea");area.value=text;
|
|
75
|
+
document.body.appendChild(area);area.select();try{document.execCommand("copy")}catch(e){}
|
|
76
|
+
document.body.removeChild(area);done()}
|
|
77
|
+
if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(text).then(done,fallback)}
|
|
78
|
+
else{fallback()}})
|
|
79
|
+
JS
|
|
80
|
+
|
|
81
|
+
# Fallback for engines without details::details-content (the print CSS
|
|
82
|
+
# does this JS-free where supported): open every details for the
|
|
83
|
+
# printout, then restore the reader's open/closed state afterwards.
|
|
84
|
+
PRINT_SCRIPT = <<~JS
|
|
85
|
+
(function(){var wasClosed=[];
|
|
86
|
+
window.addEventListener("beforeprint",function(){wasClosed=[];
|
|
87
|
+
document.querySelectorAll("details:not([open])").forEach(function(d){wasClosed.push(d);d.open=true})});
|
|
88
|
+
window.addEventListener("afterprint",function(){wasClosed.forEach(function(d){d.open=false});wasClosed=[]})})();
|
|
89
|
+
JS
|
|
90
|
+
|
|
41
91
|
# The vendored viewer bundle, embedded verbatim into every report.
|
|
42
92
|
BUNDLE_PATH = File.expand_path("assets/mutation_report.js", __dir__)
|
|
43
93
|
|
|
@@ -66,6 +116,7 @@ module Saboteur
|
|
|
66
116
|
results = report.results.to_h do |result|
|
|
67
117
|
[[Stryker.relative(result.subject.file, report.root), result.digest], result]
|
|
68
118
|
end
|
|
119
|
+
sections, collapsed = file_sections(data, names, results, report)
|
|
69
120
|
<<~HTML
|
|
70
121
|
<!DOCTYPE html>
|
|
71
122
|
<html lang="en"><head><meta charset="utf-8">
|
|
@@ -75,10 +126,12 @@ module Saboteur
|
|
|
75
126
|
<script>#{bundle}</script></head><body>
|
|
76
127
|
<h1>saboteur report</h1>
|
|
77
128
|
#{headline(report)}
|
|
129
|
+
#{scope_note(report)}
|
|
78
130
|
<mutation-test-report-app theme="dark" title-postfix="saboteur"></mutation-test-report-app>
|
|
79
131
|
<script type="application/json" id="saboteur-report-json">#{embed_json(data)}</script>
|
|
80
132
|
<script>document.querySelector("mutation-test-report-app").report =
|
|
81
|
-
JSON.parse(document.getElementById("saboteur-report-json").textContent)
|
|
133
|
+
JSON.parse(document.getElementById("saboteur-report-json").textContent);
|
|
134
|
+
#{COPY_SCRIPT}#{PRINT_SCRIPT}</script>
|
|
82
135
|
<details><summary>How the score is computed</summary>
|
|
83
136
|
#{summary(report, data)}
|
|
84
137
|
</details>
|
|
@@ -86,7 +139,7 @@ module Saboteur
|
|
|
86
139
|
<h2>Files</h2>
|
|
87
140
|
<p class="note">The viewer above and this table list the same mutants; the table adds notes,
|
|
88
141
|
covering tests and triage reasons.</p>
|
|
89
|
-
#{
|
|
142
|
+
#{collapse_note(collapsed)}#{file_listing(sections)}
|
|
90
143
|
</body></html>
|
|
91
144
|
HTML
|
|
92
145
|
end
|
|
@@ -99,11 +152,13 @@ module Saboteur
|
|
|
99
152
|
def headline(report)
|
|
100
153
|
breakdown = report.score_breakdown
|
|
101
154
|
text = "<p><strong>Covered mutation score: #{score_text(report.score)}</strong>"
|
|
102
|
-
|
|
155
|
+
if breakdown.tested.positive?
|
|
156
|
+
text << " (#{breakdown.detected}/#{breakdown.tested} #{Wording.detected_word(breakdown)}, tested subjects)"
|
|
157
|
+
end
|
|
103
158
|
untested = report.skipped_subjects.count { |_, reason| reason == ScoreTable::NO_COVERAGE }
|
|
104
159
|
if untested.positive?
|
|
105
160
|
noun = untested == 1 ? "subject" : "subjects"
|
|
106
|
-
text << " - #{untested} #{noun} not scored -
|
|
161
|
+
text << " - #{untested} #{noun} not scored - #{Wording.no_execution(untested)}"
|
|
107
162
|
end
|
|
108
163
|
"#{text}</p>"
|
|
109
164
|
end
|
|
@@ -126,13 +181,27 @@ module Saboteur
|
|
|
126
181
|
"<h2>Inline suppressions (#{items.size}, outside the score)</h2><ul>#{items.join}</ul>"
|
|
127
182
|
end
|
|
128
183
|
|
|
184
|
+
# A diff run's score covers only the changed lines; the shareable
|
|
185
|
+
# page says so, or a reader takes it for the whole project.
|
|
186
|
+
def scope_note(report)
|
|
187
|
+
return "" unless report.diff? && report.since
|
|
188
|
+
|
|
189
|
+
"<p class=\"note\">Diff run: only lines changed since #{escape(report.since)} are mutated - " \
|
|
190
|
+
"the score covers those changes, not the whole project.</p>"
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# "1 subject, 2 mutants": each count pluralizes on its own (C9).
|
|
194
|
+
def counts_note(report)
|
|
195
|
+
"#{Text.counted(report.subjects.size, 'subject')}, #{Text.counted(report.results.size, 'mutant')}"
|
|
196
|
+
end
|
|
197
|
+
|
|
129
198
|
def summary(report, data)
|
|
130
199
|
seed = data.dig(:config, :seed)
|
|
131
200
|
seed_note = seed ? ", seed #{seed}" : ""
|
|
132
201
|
counts = data[:files].values.flat_map { |entry| entry[:mutants] }.map { |mutant| mutant[:status] }.tally
|
|
133
202
|
<<~HTML
|
|
134
203
|
<p><strong>Covered mutation score: #{score_text(report.score)}</strong>
|
|
135
|
-
(#{report
|
|
204
|
+
(#{counts_note(report)}, #{format('%.1f', report.wall_time)}s#{seed_note})</p>
|
|
136
205
|
<p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
|
|
137
206
|
mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the
|
|
138
207
|
Stryker JSON may show a different score, because the schema counts NoCoverage as undetected.</p>
|
|
@@ -201,15 +270,81 @@ module Saboteur
|
|
|
201
270
|
data[:testFiles].values.flat_map { |group| group[:tests] }.to_h { |test| [test[:id], test[:name]] }
|
|
202
271
|
end
|
|
203
272
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
273
|
+
# The sections in page order; the open set is the top survivor-bearing
|
|
274
|
+
# files by survivor count, within the row budget. Returns the rendered
|
|
275
|
+
# sections and the count of survivor-bearing files left collapsed.
|
|
276
|
+
def file_sections(data, names, results, report)
|
|
277
|
+
survivors = data[:files].to_h do |file, entry|
|
|
278
|
+
[file, entry[:mutants].count { |mutant| mutant[:status] == "Survived" }]
|
|
279
|
+
end
|
|
280
|
+
attention = data[:files].to_h do |file, entry|
|
|
281
|
+
[file, entry[:mutants].count { |mutant| ATTENTION_STATUSES.include?(mutant[:status]) }]
|
|
282
|
+
end
|
|
283
|
+
open_set = open_files(data, attention)
|
|
284
|
+
collapsed = data[:files].keys.count { |file| attention[file].positive? && !open_set[file] }
|
|
285
|
+
sections = data[:files].map do |file, entry|
|
|
286
|
+
file_section(file, entry, names, results, report, survivor_count: survivors[file], open_file: open_set[file])
|
|
287
|
+
end
|
|
288
|
+
[sections, collapsed]
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# The open set: the top attention-bearing files by attention count,
|
|
292
|
+
# while the open rows stay under the budget.
|
|
293
|
+
def open_files(data, attention)
|
|
294
|
+
ranked = data[:files].keys.select { |file| attention[file].positive? }
|
|
295
|
+
.sort_by { |file| [-attention[file], file] }
|
|
296
|
+
open_set = {}
|
|
297
|
+
rows = 0
|
|
298
|
+
ranked.first(OPEN_FILE_LIMIT).each do |file|
|
|
299
|
+
break if rows >= OPEN_ROW_BUDGET
|
|
300
|
+
|
|
301
|
+
open_set[file] = true
|
|
302
|
+
rows += data[:files].fetch(file)[:mutants].size
|
|
303
|
+
end
|
|
304
|
+
open_set
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
def collapse_note(collapsed)
|
|
308
|
+
noun = collapsed == 1 ? "file" : "files"
|
|
309
|
+
if collapsed.zero?
|
|
310
|
+
""
|
|
311
|
+
else
|
|
312
|
+
"<p class=\"note print-hide\" id=\"files\">#{collapsed} #{noun} " \
|
|
313
|
+
"with survivors, timeouts or errors collapsed - " \
|
|
314
|
+
"<a href=\"#files\" onclick=\"document.querySelectorAll('details.file').forEach" \
|
|
315
|
+
"(function(d){d.open=true});return false\">expand all</a></p>\n"
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def file_section(file, entry, names, results, report, survivor_count:, open_file:)
|
|
320
|
+
open = open_file ? " open" : ""
|
|
321
|
+
# Attention rows (survivors, timeouts, errors) come first, so a big
|
|
322
|
+
# file's survivors are not buried between killed rows; position
|
|
323
|
+
# order holds within each group (#1233).
|
|
324
|
+
ordered = entry[:mutants].sort_by { |mutant| [attention_rank(mutant), *position_key(mutant)] }
|
|
207
325
|
rows = ordered.map { |mutant| mutant_row(mutant, names, results[[file, mutant[:id]]], report) }
|
|
208
|
-
"<details#{open}><summary>#{escape(file)}
|
|
326
|
+
"<details class=\"file\"#{open}><summary>#{escape(file)} " \
|
|
327
|
+
"(#{entry[:mutants].size} mutants#{survivors_note(survivor_count, open_file)})</summary>\n" \
|
|
209
328
|
"<div class=\"scroll\"><table><thead><tr><th>Location</th><th>Status</th><th>Operator</th><th>Change</th>" \
|
|
210
329
|
"<th>Id</th><th>Notes</th></tr></thead>\n#{rows.join("\n")}</table></div></details>"
|
|
211
330
|
end
|
|
212
331
|
|
|
332
|
+
# A collapsed survivor file names its survivors in the header, so the
|
|
333
|
+
# count is visible without expanding it.
|
|
334
|
+
def file_listing(sections)
|
|
335
|
+
return sections.join("\n") unless sections.empty?
|
|
336
|
+
|
|
337
|
+
"<p class=\"note\">No mutants - nothing to show.</p>"
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def survivors_note(survivor_count, open_file)
|
|
341
|
+
open_file || survivor_count.zero? ? "" : ", #{survivor_count} survivor#{'s' unless survivor_count == 1}"
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def attention_rank(mutant)
|
|
345
|
+
ATTENTION_STATUSES.include?(mutant[:status]) ? 0 : 1
|
|
346
|
+
end
|
|
347
|
+
|
|
213
348
|
def position_key(mutant)
|
|
214
349
|
start = mutant[:location][:start]
|
|
215
350
|
[start[:line], start[:column], mutant[:id]]
|
|
@@ -222,7 +357,8 @@ module Saboteur
|
|
|
222
357
|
"<tr><td class=\"fit\">#{location_cell(mutant[:location][:start], result, report)}</td>" \
|
|
223
358
|
"<td>#{row_status(mutant, result)}</td>" \
|
|
224
359
|
"<td>#{escape(mutant[:mutatorName])}</td><td class=\"wide\">#{change(mutant, result)}</td>" \
|
|
225
|
-
"<td class=\"fit\"><code>#{escape(mutant[:id])}</code
|
|
360
|
+
"<td class=\"fit\"><code>#{escape(mutant[:id])}</code>#{copy_button(mutant, seed: report.seed)}</td>" \
|
|
361
|
+
"<td class=\"wide\">#{escape(notes)}</td></tr>"
|
|
226
362
|
end
|
|
227
363
|
|
|
228
364
|
# In diff mode, the changed lines the mutant touches, under its own
|
|
@@ -251,6 +387,20 @@ module Saboteur
|
|
|
251
387
|
"covered by #{passed.size} #{passed.one? ? 'test' : 'tests'} that #{when_passed}: #{shown}#{extra}"
|
|
252
388
|
end
|
|
253
389
|
|
|
390
|
+
# Only survivors are worth replaying, so only they get the button.
|
|
391
|
+
# Every row's button reads "copy"; the aria-label names the mutant it
|
|
392
|
+
# belongs to, so a screen reader's button list stays navigable.
|
|
393
|
+
def copy_button(mutant, seed: nil)
|
|
394
|
+
return unless mutant[:status] == "Survived"
|
|
395
|
+
|
|
396
|
+
replay = escape(SurvivorGuidance.replay_command(mutant[:id], seed: seed))
|
|
397
|
+
start = mutant[:location][:start]
|
|
398
|
+
label = escape("Copy the replay command for #{mutant[:mutatorName]} " \
|
|
399
|
+
"at line #{start[:line]}, column #{start[:column]}")
|
|
400
|
+
"<button type=\"button\" class=\"copy\" data-copy=\"#{replay}\" aria-label=\"#{label}\" " \
|
|
401
|
+
'title="Copy the replay command">copy</button>'
|
|
402
|
+
end
|
|
403
|
+
|
|
254
404
|
def row_status(mutant, result)
|
|
255
405
|
badge = status_label(mutant[:status])
|
|
256
406
|
# Only survivors are ever put on the ignore list.
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../survivor_guidance"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
module Reporter
|
|
6
8
|
# The footer under a run that left survivors: how to look at one, how to
|
|
@@ -26,14 +28,14 @@ module Saboteur
|
|
|
26
28
|
if command
|
|
27
29
|
@out.puts " replay one survivor: #{@palette.paint(:info, command)}"
|
|
28
30
|
else
|
|
29
|
-
|
|
30
|
-
@out.puts " replay one survivor: "
|
|
31
|
-
|
|
32
|
-
@out.puts " (add this run's own --config, --since, --subject, --test and source options)"
|
|
31
|
+
command = SurvivorGuidance.replay_command(survivor.digest, seed: report.seed)
|
|
32
|
+
@out.puts " replay one survivor: #{@palette.paint(:info, command)}"
|
|
33
|
+
@out.puts " #{SurvivorGuidance::REPLAY_CAVEAT}"
|
|
33
34
|
end
|
|
34
|
-
@out.puts "
|
|
35
|
-
|
|
36
|
-
@out.puts "
|
|
35
|
+
@out.puts " #{SurvivorGuidance::IGNORE_INTRO}"
|
|
36
|
+
@out.puts " Saboteur.ignore do |i|"
|
|
37
|
+
@out.puts " #{SurvivorGuidance.ignore_entry(survivor.digest, survivor.subject.id)}"
|
|
38
|
+
@out.puts " end"
|
|
37
39
|
@out.puts " let an agent close the gaps: saboteur run --agent, with the same options"
|
|
38
40
|
end
|
|
39
41
|
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# One coat of paint for every warning notice: the fact lines carry the
|
|
7
|
+
# :warn token, a "Fix:" line (the two-line shape) stays :dim. Colour
|
|
8
|
+
# only - the writers own the wording, and monochrome passes the text
|
|
9
|
+
# through byte-identical.
|
|
10
|
+
module Notice
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# Paint line by line so a multi-line notice keeps its shape: any line
|
|
14
|
+
# starting with "Fix:" is the actionable tail and stays dim, every
|
|
15
|
+
# other line is the warning fact.
|
|
16
|
+
def paint(palette, text)
|
|
17
|
+
text.lines.map do |line|
|
|
18
|
+
line = line.chomp
|
|
19
|
+
palette.paint(line.start_with?("Fix:") ? :dim : :warn, line)
|
|
20
|
+
end.join("\n")
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../engine/result_cache"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Reporter
|
|
8
|
+
# Verdict-reuse feedback: named whenever the run opted into reuse, since a
|
|
9
|
+
# silent run leaves the flag's effect unknowable. A plain run (no reuse
|
|
10
|
+
# mechanism, nothing reused) stays silent. The label is the flag
|
|
11
|
+
# the user typed (--incremental or --resume, Engine#reuse_label). The
|
|
12
|
+
# denominator counts only storable verdicts (Engine::ResultCache::STORED);
|
|
13
|
+
# skips and errors are never cached, so naming them explains a short one.
|
|
14
|
+
class ReuseNote
|
|
15
|
+
def initialize(out)
|
|
16
|
+
@out = out
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def print(report)
|
|
20
|
+
return unless report.results.any? && (report.reuse_label || report.results.any?(&:reused?))
|
|
21
|
+
|
|
22
|
+
reused = report.results.count(&:reused?)
|
|
23
|
+
label = report.reuse_label || "--incremental"
|
|
24
|
+
if reused.zero?
|
|
25
|
+
@out.puts "no verdicts reused from earlier runs (#{label}): first #{label} run, " \
|
|
26
|
+
"or every mutant's inputs changed"
|
|
27
|
+
return
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
storable = report.results.count { |result| Engine::ResultCache::STORED.include?(result.status) }
|
|
31
|
+
line = "reused #{reused} of #{storable} verdicts from earlier runs (#{label})"
|
|
32
|
+
reruns = report.results.size - storable
|
|
33
|
+
@out.puts reruns.positive? ? "#{line}; #{reruns} skipped/errored mutant(s) always rerun" : line
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|