saboteur 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +399 -0
- data/README.md +202 -42
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/comparison.rb +1 -0
- data/lib/saboteur/baseline/evidence.rb +11 -1
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +49 -4
- data/lib/saboteur/cli/help_options.rb +16 -0
- data/lib/saboteur/cli/presets.rb +36 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -6
- data/lib/saboteur/cli.rb +80 -48
- data/lib/saboteur/config.rb +88 -13
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/doctor.rb +64 -2
- data/lib/saboteur/engine/baseline_guard.rb +79 -5
- data/lib/saboteur/engine/boot.rb +12 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +9 -1
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +48 -6
- data/lib/saboteur/engine/diagnostics.rb +23 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +124 -8
- data/lib/saboteur/engine/early_stop_recording.rb +46 -0
- data/lib/saboteur/engine/result_cache.rb +2 -1
- data/lib/saboteur/engine/run_budget.rb +63 -0
- data/lib/saboteur/engine/run_fingerprint.rb +5 -1
- data/lib/saboteur/engine/sandbox_policy.rb +18 -11
- data/lib/saboteur/engine/scheduler.rb +114 -18
- data/lib/saboteur/engine/subject_selection.rb +84 -5
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +183 -23
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +175 -96
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/runner.rb +179 -17
- data/lib/saboteur/execution/sandbox.rb +6 -1
- data/lib/saboteur/execution/supervisor.rb +18 -3
- data/lib/saboteur/execution/worker_pool.rb +87 -7
- data/lib/saboteur/frameworks/database_hosts.rb +30 -0
- data/lib/saboteur/frameworks/rails.rb +8 -10
- data/lib/saboteur/help.rb +55 -10
- data/lib/saboteur/ignore_list.rb +11 -1
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +40 -4
- data/lib/saboteur/mutation_source/builtin.rb +22 -13
- data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/method_swap.rb +45 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/preview.rb +12 -2
- data/lib/saboteur/reporter/agent_commands.rb +49 -16
- data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
- data/lib/saboteur/reporter/agent_summary.rb +1 -1
- data/lib/saboteur/reporter/history_policy.rb +35 -0
- data/lib/saboteur/reporter/history_record.rb +4 -5
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +19 -3
- data/lib/saboteur/reporter/stryker.rb +19 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/terminal.rb +43 -10
- data/lib/saboteur/result.rb +25 -7
- data/lib/saboteur/selection/coverage_map.rb +2 -1
- data/lib/saboteur/selection/git_diff.rb +15 -5
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +11 -0
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +16 -1
- metadata +20 -2
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "did_you_mean"
|
|
5
|
+
require_relative "operator_examples"
|
|
6
|
+
|
|
4
7
|
module Saboteur
|
|
5
8
|
module MutationSource
|
|
6
9
|
# Public names used by selection, help and inline suppression validation.
|
|
@@ -29,26 +32,51 @@ module Saboteur
|
|
|
29
32
|
"argument_removal" => "Remove a call argument",
|
|
30
33
|
"pair_removal" => "Remove a hash pair",
|
|
31
34
|
"endpoint_replacement" => "Swap explicit no-argument first and last",
|
|
32
|
-
"chain_removal" => "Remove one call from a method chain"
|
|
35
|
+
"chain_removal" => "Remove one call from a method chain",
|
|
36
|
+
"method_swap" => "Swap a method with its semantic opposite",
|
|
37
|
+
"operand_replacement" => "Replace a binary operation with one operand",
|
|
38
|
+
"interpolation_removal" => "Remove part of an interpolated string"
|
|
33
39
|
}.freeze
|
|
34
|
-
|
|
35
|
-
TIERS = { "
|
|
36
|
-
"all" => DESCRIPTIONS.keys.freeze }.freeze
|
|
40
|
+
POLICY_VERSION = 2
|
|
41
|
+
TIERS = { "all" => DESCRIPTIONS.keys.freeze }.freeze
|
|
37
42
|
|
|
38
|
-
def self.list(out)
|
|
43
|
+
def self.list(out, names: [], excluded: [])
|
|
44
|
+
active = select(names, excluded)
|
|
39
45
|
DESCRIPTIONS.each do |name, description|
|
|
40
|
-
out.puts("#{name} #{
|
|
46
|
+
out.puts("#{name} #{active.include?(name) ? 'on' : 'off'} #{description} #{OperatorExamples::EXAMPLES.fetch(name)}")
|
|
41
47
|
end
|
|
42
48
|
nil
|
|
43
49
|
end
|
|
44
50
|
|
|
51
|
+
def self.policy(names, excluded)
|
|
52
|
+
{ "version" => POLICY_VERSION, "selected" => names, "excluded" => excluded,
|
|
53
|
+
"effective" => select(names, excluded) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Exclusion wins over explicit selection; empty exclusions remove nothing.
|
|
57
|
+
def self.select(names, excluded)
|
|
58
|
+
selected = resolve(names)
|
|
59
|
+
excluded.empty? ? selected : selected - resolve(excluded)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def self.suggestion(name)
|
|
63
|
+
matches = DidYouMean::SpellChecker.new(dictionary: DESCRIPTIONS.keys).correct(name)
|
|
64
|
+
matches.empty? ? "" : "Did you mean #{matches.first}?"
|
|
65
|
+
end
|
|
66
|
+
|
|
45
67
|
def self.resolve(names)
|
|
46
|
-
return TIERS.fetch("
|
|
68
|
+
return TIERS.fetch("all") if names.empty?
|
|
47
69
|
|
|
70
|
+
legacy = names & %w[core extended]
|
|
71
|
+
unless legacy.empty?
|
|
72
|
+
raise ConfigError, "operator tiers #{legacy.join(', ')} were removed: use concrete names or all; " \
|
|
73
|
+
"for the old core set exclude range,operator_assignment,condition_negation"
|
|
74
|
+
end
|
|
48
75
|
unknown = names - DESCRIPTIONS.keys - TIERS.keys
|
|
49
76
|
unless unknown.empty?
|
|
50
77
|
raise ConfigError, "unknown operators: #{unknown.join(', ')}. Valid names: " \
|
|
51
|
-
"#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}"
|
|
78
|
+
"#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}. " \
|
|
79
|
+
"#{suggestion(unknown.first)} See --list-operators."
|
|
52
80
|
end
|
|
53
81
|
names.flat_map { |name| TIERS.fetch(name) { [name] } }.uniq
|
|
54
82
|
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module MutationSource
|
|
8
|
+
# The tree walk shared by the statement-driven operators: visit every
|
|
9
|
+
# node except nested definitions, skip arid statements, and hand each
|
|
10
|
+
# remaining node to the operator's mutate_node hook.
|
|
11
|
+
module OperatorVisit
|
|
12
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
13
|
+
|
|
14
|
+
def mutations_for(subject)
|
|
15
|
+
found = []
|
|
16
|
+
visit(subject.node.body, subject, found)
|
|
17
|
+
found
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def visit(node, subject, found)
|
|
23
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
24
|
+
|
|
25
|
+
if node.is_a?(Prism::StatementsNode)
|
|
26
|
+
node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
|
|
27
|
+
return
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
mutate_node(node, subject, found)
|
|
31
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def arid?(node)
|
|
35
|
+
@arid.any? { |pattern| pattern.match?(node.slice) }
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
data/lib/saboteur/preview.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "engine"
|
|
5
|
+
require_relative "text"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
# Source-only planning. Never boots the host or executes its test suite.
|
|
@@ -13,6 +14,7 @@ module Saboteur
|
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
def call
|
|
17
|
+
announce_operators
|
|
16
18
|
selection = Engine::SubjectSelection.new(@config, @diagnostics)
|
|
17
19
|
subjects = selection.select(@config.included_files)
|
|
18
20
|
scope = selection.changed_lines && Engine::DiffScope.new(selection.changed_lines)
|
|
@@ -24,12 +26,20 @@ module Saboteur
|
|
|
24
26
|
@out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
|
|
25
27
|
"dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
|
|
26
28
|
incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
|
|
27
|
-
|
|
29
|
+
print_warnings
|
|
28
30
|
nil
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
private
|
|
32
34
|
|
|
35
|
+
def print_warnings
|
|
36
|
+
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def announce_operators
|
|
40
|
+
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
|
|
41
|
+
end
|
|
42
|
+
|
|
33
43
|
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
34
44
|
# Name that scope rather than implying these rows were executed or scored.
|
|
35
45
|
def inline_suppressions
|
|
@@ -83,7 +93,7 @@ module Saboteur
|
|
|
83
93
|
# coverage map means "no estimate", never an exception.
|
|
84
94
|
def cached_map
|
|
85
95
|
path = File.join(@config.path("cache_dir"), "preview.json")
|
|
86
|
-
receipt = JSON.parse(File.read(path))
|
|
96
|
+
receipt = JSON.parse(Text.utf8_strict(File.read(path)))
|
|
87
97
|
return unless receipt["fingerprint"] == Engine::RunFingerprint.new(config: @config).snapshot.digest
|
|
88
98
|
|
|
89
99
|
Selection::CoverageMap.load(receipt["path"])
|
|
@@ -8,14 +8,26 @@ module Saboteur
|
|
|
8
8
|
module Agent
|
|
9
9
|
# The shell commands an agent is told to run next. Every word is escaped with
|
|
10
10
|
# Shellwords, so a ref, path or subject holding shell syntax stays one literal
|
|
11
|
-
# argument. A command is never cut short: when it is
|
|
12
|
-
# is left out, because half a command could run
|
|
11
|
+
# argument. A command for the agent protocol is never cut short: when it is
|
|
12
|
+
# longer than MAX_BYTES it is left out, because half a command could run
|
|
13
|
+
# something else. The text footer passes capped: false instead - there the
|
|
14
|
+
# promise is the complete command, however long (#261).
|
|
13
15
|
module Commands
|
|
14
16
|
MAX_BYTES = 1024
|
|
15
17
|
|
|
16
18
|
# Options typed on the command line that change how a run behaves.
|
|
17
19
|
SETTING_FLAGS = { reeval: "--reeval", integration: "--integration", jobs: "--jobs",
|
|
18
|
-
timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html"
|
|
20
|
+
timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html",
|
|
21
|
+
operators: "--operators", exclude_operators: "--exclude-operators",
|
|
22
|
+
baseline_passes: "--baseline-passes", max_duration: "--max-duration" }.freeze
|
|
23
|
+
|
|
24
|
+
# CLI-only boolean switches recorded in settings, mapped by value, so
|
|
25
|
+
# an explicit false (--no-max-duration-ok over a config true) survives
|
|
26
|
+
# into the generated commands instead of reverting to the config.
|
|
27
|
+
BOOLEAN_SETTING_FLAGS = { allow_hazardous: { true => "--allow-hazardous" },
|
|
28
|
+
allow_focus: { true => "--allow-focus" },
|
|
29
|
+
max_duration_ok: { true => "--max-duration-ok",
|
|
30
|
+
false => "--no-max-duration-ok" } }.freeze
|
|
19
31
|
|
|
20
32
|
class << self
|
|
21
33
|
# Run the same selection again: ref, subjects, sources, tests, config,
|
|
@@ -25,38 +37,59 @@ module Saboteur
|
|
|
25
37
|
finish(words, provenance)
|
|
26
38
|
end
|
|
27
39
|
|
|
28
|
-
# Replay one survivor by digest in the same selection.
|
|
29
|
-
|
|
40
|
+
# Replay one survivor by digest in the same selection. capped: false
|
|
41
|
+
# skips the protocol byte cap, for callers that print to a terminal.
|
|
42
|
+
def replay(provenance, digest, capped: true)
|
|
30
43
|
words = ["saboteur", "run", "--id", digest, *selection(provenance, replay: true), "--seed",
|
|
31
44
|
provenance.seed.to_s]
|
|
32
|
-
finish(words, provenance)
|
|
45
|
+
finish(words, provenance, capped: capped)
|
|
33
46
|
end
|
|
34
47
|
|
|
35
48
|
private
|
|
36
49
|
|
|
37
|
-
def finish(words, provenance)
|
|
50
|
+
def finish(words, provenance, capped: true)
|
|
38
51
|
words += ["--", *provenance.sources] unless provenance.sources.empty?
|
|
39
52
|
command = Shellwords.join(words)
|
|
40
|
-
command
|
|
53
|
+
command if !capped || command.bytesize <= MAX_BYTES
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def mode_words(provenance)
|
|
57
|
+
words = []
|
|
58
|
+
words << "--no-cache" if provenance.cache == false
|
|
59
|
+
words << "--incremental" if provenance.incremental
|
|
60
|
+
words.concat(boolean_words(provenance))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def boolean_words(provenance)
|
|
64
|
+
BOOLEAN_SETTING_FLAGS.filter_map { |key, words| words[provenance.settings[key]] }
|
|
41
65
|
end
|
|
42
66
|
|
|
43
|
-
# An explicit scope is frozen
|
|
44
|
-
#
|
|
67
|
+
# An explicit scope is frozen as --test, which wins over inferred
|
|
68
|
+
# selection by precedence, whatever the config file sets.
|
|
45
69
|
def test_words(tests)
|
|
46
|
-
tests.
|
|
70
|
+
tests.flat_map { |test| ["--test", test] }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def setting_words(provenance)
|
|
74
|
+
SETTING_FLAGS.flat_map do |key, flag|
|
|
75
|
+
next [] unless provenance.settings.key?(key)
|
|
76
|
+
|
|
77
|
+
value = provenance.settings[key]
|
|
78
|
+
[flag, value.is_a?(Array) ? value.join(",") : value.to_s]
|
|
79
|
+
end
|
|
47
80
|
end
|
|
48
81
|
|
|
82
|
+
# Whole-suite scope is a bare --all, only when no tests resolved;
|
|
83
|
+
# --all never combines with --test.
|
|
49
84
|
def selection(provenance, replay: false)
|
|
50
85
|
words = []
|
|
51
86
|
words.push("--config", provenance.config_path) if provenance.config_path
|
|
52
87
|
words.push("--since", provenance.ref) if provenance.ref
|
|
53
88
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
89
|
+
words << "--all" if provenance.settings[:all] && provenance.tests.empty?
|
|
54
90
|
words.push(*test_words(provenance.tests))
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
end
|
|
58
|
-
words << "--no-cache" if provenance.cache == false
|
|
59
|
-
words << "--incremental" if provenance.incremental
|
|
91
|
+
words.concat(setting_words(provenance))
|
|
92
|
+
words.concat(mode_words(provenance))
|
|
60
93
|
words.push("--id", provenance.digest) if provenance.digest && !replay
|
|
61
94
|
words
|
|
62
95
|
end
|
|
@@ -8,6 +8,8 @@ module Saboteur
|
|
|
8
8
|
# stable code, a count and a plain meaning.
|
|
9
9
|
module Incomplete
|
|
10
10
|
NO_COVERAGE = "no coverage"
|
|
11
|
+
SYNTAX_FAILURE = "invalid_file_syntax"
|
|
12
|
+
UNREADABLE_FAILURE = "unreadable"
|
|
11
13
|
|
|
12
14
|
class << self
|
|
13
15
|
def reasons(report)
|
|
@@ -23,14 +25,37 @@ module Saboteur
|
|
|
23
25
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
24
26
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
25
27
|
add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
|
|
26
|
-
add(reasons, "
|
|
27
|
-
"
|
|
28
|
+
add(reasons, "unverified", report.counts[:unverified],
|
|
29
|
+
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
30
|
+
discovery_failures(reasons, report)
|
|
31
|
+
early_stop_reason(reasons, report)
|
|
28
32
|
inline_scope(reasons, report)
|
|
29
33
|
reasons
|
|
30
34
|
end
|
|
31
35
|
|
|
32
36
|
private
|
|
33
37
|
|
|
38
|
+
# Source the selection could not read at all: its subjects never
|
|
39
|
+
# entered the run, so a clean or complete verdict would be wrong.
|
|
40
|
+
def discovery_failures(reasons, report)
|
|
41
|
+
failures = report.discovery_failures
|
|
42
|
+
add(reasons, SYNTAX_FAILURE, failures.count { |f| f.reason == SYNTAX_FAILURE },
|
|
43
|
+
"included files did not parse; their subjects were not discovered")
|
|
44
|
+
add(reasons, UNREADABLE_FAILURE, failures.count { |f| f.reason == UNREADABLE_FAILURE },
|
|
45
|
+
"included untracked files could not be read; their subjects were not covered")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def early_stop_reason(reasons, report)
|
|
49
|
+
stop = report.early_stop
|
|
50
|
+
return unless stop
|
|
51
|
+
|
|
52
|
+
label = stop.reason == "fail_fast" ? "--fail-fast stopped" : "the --max-duration budget stopped"
|
|
53
|
+
add(reasons, stop.reason, stop.unreached.size, "selected subjects did not run because #{label} the run")
|
|
54
|
+
omitted = stop.truncated.sum { |entry| entry[:omitted] }
|
|
55
|
+
add(reasons, "#{stop.reason}_truncated", omitted,
|
|
56
|
+
"mutants omitted inside subjects that stopped mid-way because #{label} the run")
|
|
57
|
+
end
|
|
58
|
+
|
|
34
59
|
def inline_scope(reasons, report)
|
|
35
60
|
add(reasons, "inline_suppressed", report.inline_suppressions.size,
|
|
36
61
|
"candidates were suppressed by inline directives; see the full report audit")
|
|
@@ -22,7 +22,7 @@ module Saboteur
|
|
|
22
22
|
verdict = verdict_of(report, status)
|
|
23
23
|
{
|
|
24
24
|
version: VERSION, status: status, verdict: verdict, exitCode: exit_code_for(verdict),
|
|
25
|
-
provenance: provenance_hash(provenance),
|
|
25
|
+
provenance: provenance_hash(provenance).merge(operatorPolicy: report.operator_policy),
|
|
26
26
|
counts: counts(report), score: score(report),
|
|
27
27
|
scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
|
|
28
28
|
incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# v1 stays readable, but v2 records require versioned operator evidence.
|
|
7
|
+
module HistoryPolicy
|
|
8
|
+
def validate_header(record, fields)
|
|
9
|
+
version = record.is_a?(Hash) && record["version"]
|
|
10
|
+
raise Error, "score history contains an invalid version" unless version.is_a?(Integer) && [1,
|
|
11
|
+
2].include?(version)
|
|
12
|
+
|
|
13
|
+
fields(record, version == 2 ? fields + ["operatorPolicy"] : fields)
|
|
14
|
+
validate_policy(record["operatorPolicy"]) if version == 2
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def validate_policy(value)
|
|
18
|
+
raise Error, "score history contains invalid operator policy" unless value.is_a?(Hash)
|
|
19
|
+
|
|
20
|
+
fields(value, %w[version selected excluded effective])
|
|
21
|
+
raise Error, "score history contains invalid operator policy" if value["effective"].empty?
|
|
22
|
+
|
|
23
|
+
lists = value.values_at("selected", "excluded", "effective")
|
|
24
|
+
unless lists.all? { |list| list.is_a?(Array) && list.all?(String) }
|
|
25
|
+
raise Error, "score history contains invalid operator policy"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
expected = MutationSource::OperatorRegistry.policy(value["selected"], value["excluded"])
|
|
29
|
+
raise Error, "score history contains invalid operator policy" unless value == expected
|
|
30
|
+
rescue ConfigError, NoMethodError, TypeError
|
|
31
|
+
raise Error, "score history contains invalid operator policy"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "time"
|
|
6
|
+
require_relative "history_policy"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
module Reporter
|
|
@@ -21,6 +22,8 @@ module Saboteur
|
|
|
21
22
|
end
|
|
22
23
|
end
|
|
23
24
|
|
|
25
|
+
extend HistoryPolicy
|
|
26
|
+
|
|
24
27
|
module_function
|
|
25
28
|
|
|
26
29
|
def parse(line)
|
|
@@ -32,11 +35,7 @@ module Saboteur
|
|
|
32
35
|
end
|
|
33
36
|
|
|
34
37
|
def validate(record)
|
|
35
|
-
|
|
36
|
-
unless record.fetch("version").is_a?(Integer) && record.fetch("version") == 1
|
|
37
|
-
raise Error,
|
|
38
|
-
"score history contains an invalid version"
|
|
39
|
-
end
|
|
38
|
+
validate_header(record, FIELDS)
|
|
40
39
|
|
|
41
40
|
date(record.fetch("date"))
|
|
42
41
|
commit = record.fetch("commit")
|
|
@@ -7,9 +7,14 @@ module Saboteur
|
|
|
7
7
|
# accept one, and how to hand the run to an agent. Silent when nothing
|
|
8
8
|
# survived, so a clean run stays short.
|
|
9
9
|
class NextSteps
|
|
10
|
-
|
|
10
|
+
# replay_command: digest -> the full, copy-pasteable replay command
|
|
11
|
+
# carrying the run's own selection options, however long it gets
|
|
12
|
+
# (#261). When no builder is wired the digest-only hint (with its
|
|
13
|
+
# reassembly note) is the fallback.
|
|
14
|
+
def initialize(out, palette, replay_command: nil)
|
|
11
15
|
@out = out
|
|
12
16
|
@palette = palette
|
|
17
|
+
@replay_command = replay_command
|
|
13
18
|
end
|
|
14
19
|
|
|
15
20
|
def call(report)
|
|
@@ -17,9 +22,14 @@ module Saboteur
|
|
|
17
22
|
return unless survivor
|
|
18
23
|
|
|
19
24
|
@out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
25
|
+
command = @replay_command&.call(survivor.digest)
|
|
26
|
+
if command
|
|
27
|
+
@out.puts " replay one survivor: #{command}"
|
|
28
|
+
else
|
|
29
|
+
seed = " --seed #{report.seed}" if report.seed
|
|
30
|
+
@out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
|
|
31
|
+
@out.puts " (add this run's own --config, --since, --subject, --test and source options)"
|
|
32
|
+
end
|
|
23
33
|
@out.puts " accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
|
|
24
34
|
entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
|
|
25
35
|
@out.puts " #{entry}"
|
|
@@ -13,6 +13,7 @@ module Saboteur
|
|
|
13
13
|
# Opt-in append-only score observations, not a baseline or a CI gate.
|
|
14
14
|
class ScoreHistory
|
|
15
15
|
DIFFERENT_RUN = "score history: change unavailable (different selection/inventory or incomplete run)"
|
|
16
|
+
DIFFERENT_POLICY = "score history: change unavailable (different record version or operator policy)"
|
|
16
17
|
MISSING_SCORE = "score history: change unavailable (no tested mutants)"
|
|
17
18
|
MAX_RECORD_BYTES = 64 * 1024
|
|
18
19
|
OPEN_FLAGS = File::RDWR | File::CREAT | File::NONBLOCK | File::NOFOLLOW
|
|
@@ -38,7 +39,11 @@ module Saboteur
|
|
|
38
39
|
file.flush
|
|
39
40
|
change(previous, record)
|
|
40
41
|
end
|
|
41
|
-
|
|
42
|
+
# SystemCallError/IOError from the write itself propagate unwrapped on
|
|
43
|
+
# purpose: the CLI treats those as a soft artifact failure (warn and
|
|
44
|
+
# exit by the results), while validation, integrity, record-size and
|
|
45
|
+
# lock failures stay hard errors (#296).
|
|
46
|
+
rescue JSON::ParserError, KeyError, TypeError => e
|
|
42
47
|
raise Error, "score history was not recorded: #{e.message}"
|
|
43
48
|
end
|
|
44
49
|
|
|
@@ -58,13 +63,13 @@ module Saboteur
|
|
|
58
63
|
private
|
|
59
64
|
|
|
60
65
|
def build(report, head, scope)
|
|
61
|
-
{ "version" =>
|
|
66
|
+
{ "version" => 2, "date" => Time.now.utc.iso8601, "commit" => head,
|
|
62
67
|
"selection" => Digest::SHA256.hexdigest(JSON.generate(scope.sort_by { |key, _| key.to_s }.to_h)),
|
|
63
68
|
"inventory" => Digest::SHA256.hexdigest(JSON.generate(report.results.map(&:digest).sort)),
|
|
64
69
|
"score" => overall_score(report), "coveredScore" => report.score,
|
|
65
70
|
"counts" => report.counts.transform_keys(&:to_s),
|
|
66
71
|
"breakdown" => report.score_breakdown.to_h.transform_keys(&:to_s),
|
|
67
|
-
"incomplete" => incompleteness(report) }
|
|
72
|
+
"incomplete" => incompleteness(report), "operatorPolicy" => report.operator_policy }
|
|
68
73
|
end
|
|
69
74
|
|
|
70
75
|
def overall_score(report)
|
|
@@ -107,11 +112,22 @@ module Saboteur
|
|
|
107
112
|
previous
|
|
108
113
|
end
|
|
109
114
|
|
|
115
|
+
# Only one record version and one non-null operator policy compare:
|
|
116
|
+
# scores never mix across v1/v2 or across policy configurations.
|
|
117
|
+
# A non-null previous policy forces a non-null equal current one, so
|
|
118
|
+
# only the previous record needs the null guard.
|
|
119
|
+
def comparable_policy?(previous, current)
|
|
120
|
+
previous["version"] == current["version"] &&
|
|
121
|
+
!previous["operatorPolicy"].nil? &&
|
|
122
|
+
previous["operatorPolicy"] == current["operatorPolicy"]
|
|
123
|
+
end
|
|
124
|
+
|
|
110
125
|
def change(previous, current)
|
|
111
126
|
return "score history: first record" unless previous
|
|
112
127
|
return DIFFERENT_RUN unless
|
|
113
128
|
previous["selection"] == current["selection"] && previous["inventory"] == current["inventory"] &&
|
|
114
129
|
previous["incomplete"].empty? && current["incomplete"].empty?
|
|
130
|
+
return DIFFERENT_POLICY unless comparable_policy?(previous, current)
|
|
115
131
|
return MISSING_SCORE if
|
|
116
132
|
previous["coveredScore"].nil? || current["coveredScore"].nil?
|
|
117
133
|
|
|
@@ -40,13 +40,15 @@ module Saboteur
|
|
|
40
40
|
warnings: report.warnings,
|
|
41
41
|
framework: { name: "saboteur", version: VERSION },
|
|
42
42
|
saboteur: { findings: Findings.build(report), scores: scores(report),
|
|
43
|
-
inlineSuppressions: report
|
|
44
|
-
|
|
45
|
-
end }
|
|
46
|
-
.merge(exit_cause ? { exitCause: exit_cause } : {})
|
|
43
|
+
inlineSuppressions: inline_audit(report) }
|
|
44
|
+
.merge(operatorPolicy: report.operator_policy).merge(exit_cause ? { exitCause: exit_cause } : {})
|
|
47
45
|
}.merge(report.seed ? { config: { seed: report.seed } } : {})
|
|
48
46
|
end
|
|
49
47
|
|
|
48
|
+
def self.inline_audit(report)
|
|
49
|
+
report.inline_suppressions.map { |entry| entry.merge(file: relative(entry[:file], report.root)) }
|
|
50
|
+
end
|
|
51
|
+
|
|
50
52
|
# Score with its assertion and crash shares, for all mutants and without
|
|
51
53
|
# the whole-body ones (body_nil, body_true, body_false, body_raise).
|
|
52
54
|
def self.scores(report)
|
|
@@ -56,7 +58,14 @@ module Saboteur
|
|
|
56
58
|
# nil for a run that covered its whole selection.
|
|
57
59
|
def self.early_stop(report)
|
|
58
60
|
stop = report.early_stop
|
|
59
|
-
|
|
61
|
+
return unless stop
|
|
62
|
+
|
|
63
|
+
{ reason: stop.reason, unreachedSubjects: stop.unreached, truncatedSubjects: truncated_subjects(stop),
|
|
64
|
+
unverifiedSubjects: stop.unverified }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def self.truncated_subjects(stop)
|
|
68
|
+
stop.truncated.map { |entry| { subject: entry[:subject], omittedMutants: entry[:omitted] } }
|
|
60
69
|
end
|
|
61
70
|
|
|
62
71
|
def self.skipped_subjects(report)
|
|
@@ -113,8 +122,9 @@ module Saboteur
|
|
|
113
122
|
when :error, :resource_limit then "RuntimeError"
|
|
114
123
|
when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
|
|
115
124
|
# The schema has no status for "ran, but no test can observe a
|
|
116
|
-
# change here"
|
|
117
|
-
|
|
125
|
+
# change here", nor for a verdict the budget left unchecked.
|
|
126
|
+
# Ignored with the reason is the honest mapping.
|
|
127
|
+
when :ran_at_load, :unreachable, :unverified then "Ignored"
|
|
118
128
|
end
|
|
119
129
|
end
|
|
120
130
|
|
|
@@ -123,7 +133,8 @@ module Saboteur
|
|
|
123
133
|
return "resource limit: #{result.reason}" if result.status == :resource_limit
|
|
124
134
|
return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
|
|
125
135
|
|
|
126
|
-
result.reason.to_s.
|
|
136
|
+
reason = result.reason.to_s.strip
|
|
137
|
+
reason.empty? ? nil : reason
|
|
127
138
|
end
|
|
128
139
|
|
|
129
140
|
def self.location(edit, source)
|
|
@@ -19,6 +19,9 @@ module Saboteur
|
|
|
19
19
|
"safe_navigation" => "try an example where the receiver is nil",
|
|
20
20
|
"argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
|
|
21
21
|
"pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
|
|
22
|
+
"chain_removal" => "try the shorter chain's value against the longer one; if you confirm the " \
|
|
23
|
+
"shorter chain is equivalent across the intended inputs (for example a duplicated " \
|
|
24
|
+
"column), simplify the code or ignore the mutant",
|
|
22
25
|
"endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
|
|
23
26
|
"filter_replacement" => "try items that match and items that do not, to tell select from reject"
|
|
24
27
|
}.freeze
|
|
@@ -17,12 +17,12 @@ module Saboteur
|
|
|
17
17
|
MAX_ERRORS = 10
|
|
18
18
|
MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
|
|
19
19
|
|
|
20
|
-
def initialize(out, verbose: false, color: :auto)
|
|
20
|
+
def initialize(out, verbose: false, color: :auto, replay_command: nil)
|
|
21
21
|
@out = out
|
|
22
22
|
@verbose = verbose
|
|
23
23
|
@palette = Palette.for(out, mode: color)
|
|
24
24
|
@score_table = ScoreTable.new(out, @palette)
|
|
25
|
-
@next_steps = NextSteps.new(out, @palette)
|
|
25
|
+
@next_steps = NextSteps.new(out, @palette, replay_command: replay_command)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def call(report)
|
|
@@ -34,11 +34,20 @@ module Saboteur
|
|
|
34
34
|
warnings(report)
|
|
35
35
|
early_stop(report)
|
|
36
36
|
test_matrix(report)
|
|
37
|
+
operator_policy(report)
|
|
37
38
|
summary(report)
|
|
38
39
|
end
|
|
39
40
|
|
|
40
41
|
private
|
|
41
42
|
|
|
43
|
+
def operator_policy(report)
|
|
44
|
+
policy = report.operator_policy
|
|
45
|
+
return unless policy && policy["excluded"].any?
|
|
46
|
+
|
|
47
|
+
@out.puts("Excluded operators: #{policy['excluded'].join(', ')} " \
|
|
48
|
+
"(policy #{policy['version']}; #{policy['effective'].size} enabled)")
|
|
49
|
+
end
|
|
50
|
+
|
|
42
51
|
def test_matrix(report)
|
|
43
52
|
return unless report.matrix?
|
|
44
53
|
|
|
@@ -100,6 +109,7 @@ module Saboteur
|
|
|
100
109
|
"resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
|
|
101
110
|
line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
|
|
102
111
|
line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
|
|
112
|
+
line += " unverified #{c[:unverified]}" if c[:unverified].positive?
|
|
103
113
|
line
|
|
104
114
|
end
|
|
105
115
|
|
|
@@ -214,17 +224,39 @@ module Saboteur
|
|
|
214
224
|
end
|
|
215
225
|
end
|
|
216
226
|
|
|
217
|
-
# The verdict covers only the
|
|
227
|
+
# The verdict covers only the completed work, so say it where the
|
|
218
228
|
# score is read.
|
|
219
229
|
def early_stop(report)
|
|
220
230
|
stop = report.early_stop
|
|
221
231
|
return unless stop
|
|
222
232
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
"covers only
|
|
226
|
-
stop.unreached
|
|
227
|
-
|
|
233
|
+
label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
|
|
234
|
+
@out.puts @palette.paint(:yellow, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
|
|
235
|
+
"covers only completed work")
|
|
236
|
+
list(stop.unreached) { |id| " not run: #{id}" }
|
|
237
|
+
list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
|
|
238
|
+
list(stop.unverified) { |id| " unverified: #{id}" }
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def list(entries)
|
|
242
|
+
entries.first(MAX_ERRORS).each { |entry| @out.puts yield(entry) }
|
|
243
|
+
@out.puts " ... #{entries.size - MAX_ERRORS} more" if entries.size > MAX_ERRORS
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def early_stop_summary(stop)
|
|
247
|
+
parts = []
|
|
248
|
+
if stop.unreached.any?
|
|
249
|
+
parts << "#{stop.unreached.size} selected subject#{'s' unless stop.unreached.one?} did not run"
|
|
250
|
+
end
|
|
251
|
+
if stop.truncated.any?
|
|
252
|
+
omitted = stop.truncated.sum { |entry| entry[:omitted] }
|
|
253
|
+
parts << "#{stop.truncated.size} subject#{'s' unless stop.truncated.one?} stopped mid-way " \
|
|
254
|
+
"(#{omitted} mutants omitted)"
|
|
255
|
+
end
|
|
256
|
+
if stop.unverified.any?
|
|
257
|
+
parts << "#{stop.unverified.size} subject#{'s' unless stop.unverified.one?} left with unverified results"
|
|
258
|
+
end
|
|
259
|
+
parts.join("; ")
|
|
228
260
|
end
|
|
229
261
|
|
|
230
262
|
def warnings(report)
|
|
@@ -237,8 +269,9 @@ module Saboteur
|
|
|
237
269
|
|
|
238
270
|
@out.puts "\nErrors (not counted as killed):"
|
|
239
271
|
errored.first(MAX_ERRORS).each do |result|
|
|
240
|
-
|
|
241
|
-
|
|
272
|
+
lines = result.reason.to_s.lines.map(&:strip)
|
|
273
|
+
@out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: #{lines.first}"
|
|
274
|
+
lines.drop(1).each { |line| @out.puts " #{line}" }
|
|
242
275
|
end
|
|
243
276
|
@out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
|
|
244
277
|
end
|