saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
data/exe/saboteur
CHANGED
|
@@ -2,15 +2,42 @@
|
|
|
2
2
|
# SPDX-License-Identifier: MIT
|
|
3
3
|
# frozen_string_literal: true
|
|
4
4
|
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Coverage comes first because it only measures files loaded after it -
|
|
6
|
+
# the crash reporter must stay measurable - and it loads inside the same
|
|
7
|
+
# guarded begin as the reporter: every later failure, even while the CLI
|
|
8
|
+
# is being required, gets the full report (the bounded one-liner, the log
|
|
9
|
+
# file, the --verbose backtrace and agent JSON when the CLI loaded far
|
|
10
|
+
# enough). If even the reporter cannot load, a bare normalized one-liner
|
|
11
|
+
# (one line, invalid bytes scrubbed, bounded length) still exits 3.
|
|
12
|
+
begin
|
|
13
|
+
require "coverage"
|
|
14
|
+
Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
|
|
15
|
+
|
|
16
|
+
require "saboteur/crash_report"
|
|
17
|
+
rescue LoadError, StandardError => e
|
|
18
|
+
message = e.message.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?").gsub(/\s+/, " ").strip
|
|
19
|
+
message = "#{message[0...160]}..." if message.length > 160
|
|
20
|
+
warn "saboteur: internal error (#{e.class}: #{message}) - please report it: " \
|
|
21
|
+
"https://github.com/vamsipavanmahesh/saboteur/issues"
|
|
22
|
+
exit 3
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# For `run` coverage counts from here: what executes while the app and
|
|
7
26
|
# spec files load is how ran-at-load mutants are told apart from uncovered
|
|
8
27
|
# ones. The engine suspends counting right after boot and reads the
|
|
9
28
|
# counters as the map's load lines (a forked child would not inherit them).
|
|
10
|
-
|
|
11
|
-
Coverage.
|
|
12
|
-
Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
|
|
29
|
+
begin
|
|
30
|
+
Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
|
|
13
31
|
|
|
14
|
-
require "saboteur/cli"
|
|
32
|
+
require "saboteur/cli"
|
|
15
33
|
|
|
16
|
-
exit Saboteur::CLI.new(ARGV).call
|
|
34
|
+
exit Saboteur::CLI.new(ARGV).call
|
|
35
|
+
rescue LoadError, StandardError => e
|
|
36
|
+
# Agent mode is visible in the raw arguments; the CLI may never have
|
|
37
|
+
# loaded, so this scan must not depend on it.
|
|
38
|
+
agent = ARGV.include?("--agent") || ARGV.include?("--format=agent") ||
|
|
39
|
+
ARGV.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
|
|
40
|
+
exit Saboteur::CrashReport.call(e, verbose: ARGV.include?("--verbose"), agent: agent,
|
|
41
|
+
err: $stderr, out: $stdout,
|
|
42
|
+
log_path: File.join(Dir.pwd, ".saboteur", "internal-error.log"))
|
|
43
|
+
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require_relative "acceptance"
|
|
5
5
|
require_relative "comparison"
|
|
6
6
|
require_relative "../engine"
|
|
7
|
+
require_relative "../text"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
9
10
|
module Baseline
|
|
@@ -70,7 +71,7 @@ module Saboteur
|
|
|
70
71
|
|
|
71
72
|
bytes = File.read(path)
|
|
72
73
|
@baseline_digest = Digest::SHA256.hexdigest(bytes)
|
|
73
|
-
manifest = Manifest.new(JSON.parse(bytes))
|
|
74
|
+
manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
|
|
74
75
|
metadata = manifest.to_h.fetch("metadata")
|
|
75
76
|
candidates = manifest.candidates.values
|
|
76
77
|
restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "tempfile"
|
|
6
6
|
|
|
7
|
+
require_relative "text"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
# Two fresh supervised children check the full suite without changing methods
|
|
9
11
|
# or writing caches. This samples stability; it cannot prove absence of flakes.
|
|
@@ -49,8 +51,9 @@ module Saboteur
|
|
|
49
51
|
|
|
50
52
|
def check_run(number, expected)
|
|
51
53
|
Tempfile.create("saboteur-baseline") do |log|
|
|
52
|
-
result = @supervisor.run(timeout: @config["baseline_timeout"]
|
|
53
|
-
|
|
54
|
+
result = @supervisor.run(timeout: @config["baseline_timeout"],
|
|
55
|
+
label: "baseline child died during setup/run") { |writer| record(writer, log.path) }
|
|
56
|
+
data = read_result(result, number, log)
|
|
54
57
|
verify_execution(data, number, expected)
|
|
55
58
|
raise Error, "baseline run #{number} left processes running; fix suite cleanup" if result.leftover_pids.any?
|
|
56
59
|
|
|
@@ -64,9 +67,12 @@ module Saboteur
|
|
|
64
67
|
raise Error, "baseline check failed: #{e.class}: #{e.message}; check suite hooks and setup"
|
|
65
68
|
end
|
|
66
69
|
|
|
67
|
-
def read_result(result, number)
|
|
70
|
+
def read_result(result, number, log = nil)
|
|
68
71
|
raise Error, "baseline run #{number} stopped: #{result.detail}" if result.stopped_by
|
|
69
|
-
|
|
72
|
+
if result.payload.empty?
|
|
73
|
+
raise Error, "baseline child died during setup/run: baseline run #{number} exited without a " \
|
|
74
|
+
"result#{exit_detail(result.exit_status)}; check suite boot and exits#{log_tail(log)}"
|
|
75
|
+
end
|
|
70
76
|
if result.child_failed?
|
|
71
77
|
raise Error, "baseline run #{number} exited unsuccessfully; fix suite cleanup and exit handlers"
|
|
72
78
|
end
|
|
@@ -77,6 +83,30 @@ module Saboteur
|
|
|
77
83
|
data
|
|
78
84
|
end
|
|
79
85
|
|
|
86
|
+
# The child's own output when it died without answering: the redirect
|
|
87
|
+
# puts its last words in this log, not on the terminal (#289). Bounded
|
|
88
|
+
# (about 2 KB, 20 lines) and scrubbed before it joins the error. Read
|
|
89
|
+
# through the parent's open handle, and strictly best-effort: a dead
|
|
90
|
+
# handle must never mask the real exit or signal error.
|
|
91
|
+
def log_tail(log)
|
|
92
|
+
return "" unless log
|
|
93
|
+
|
|
94
|
+
log.seek([log.size - 2048, 0].max)
|
|
95
|
+
tail = Text.utf8(log.read(2048).to_s).lines.last(20).map(&:chomp).join(" / ")
|
|
96
|
+
tail.empty? ? "" : "; child log tail: #{tail}"
|
|
97
|
+
rescue SystemCallError, IOError
|
|
98
|
+
""
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The child's ending when the supervisor reaped one ("exit 1",
|
|
102
|
+
# "signal SIGKILL", ...): an empty result's cause is usually there (#255).
|
|
103
|
+
def exit_detail(status)
|
|
104
|
+
return "" unless status.is_a?(Process::Status)
|
|
105
|
+
return " (signal #{Signal.signame(status.termsig)})" if status.signaled?
|
|
106
|
+
|
|
107
|
+
" (exit #{status.exitstatus})"
|
|
108
|
+
end
|
|
109
|
+
|
|
80
110
|
def verify_execution(data, number, expected)
|
|
81
111
|
unless data["passed"] == true
|
|
82
112
|
raise Error, "baseline run #{number} failed: #{data['error'] || 'fix the unchanged suite'}"
|
|
@@ -93,8 +123,12 @@ module Saboteur
|
|
|
93
123
|
exception = example.execution_result.exception
|
|
94
124
|
next unless exception
|
|
95
125
|
|
|
96
|
-
|
|
97
|
-
|
|
126
|
+
# Exception text and example names can arrive locale-tagged or as
|
|
127
|
+
# raw path bytes (e.g. Errno::EILSEQ): scrub each field before any
|
|
128
|
+
# strip, truncation or interpolation - all three raise on bad tags.
|
|
129
|
+
# This detail ships in a JSON payload, so it must be valid UTF-8.
|
|
130
|
+
message = Text.utf8(exception.message.to_s).lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
|
|
131
|
+
"#{Text.utf8(example.id.to_s)}: #{Text.utf8(example.full_description.to_s)[0, 200]}: #{message}"
|
|
98
132
|
end.first(5).join("; ")
|
|
99
133
|
end
|
|
100
134
|
|
|
@@ -8,7 +8,28 @@ module Saboteur
|
|
|
8
8
|
module ExitStatus
|
|
9
9
|
private
|
|
10
10
|
|
|
11
|
+
# Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
|
|
12
|
+
def internal_error(error)
|
|
13
|
+
CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# The crash log lives with the other run state. A broken config (or a
|
|
17
|
+
# nil cache_dir) must not swallow the crash report itself, so fall
|
|
18
|
+
# back to no log.
|
|
19
|
+
def crash_log_path
|
|
20
|
+
dir = config.path("cache_dir")
|
|
21
|
+
dir && File.join(dir, CrashReport::LOG_NAME)
|
|
22
|
+
rescue StandardError
|
|
23
|
+
nil
|
|
24
|
+
end
|
|
25
|
+
|
|
11
26
|
def exit_code(report)
|
|
27
|
+
return EXIT_BUDGET if budget_exhausted?(report)
|
|
28
|
+
|
|
29
|
+
return EXIT_TOOL if untested_partial?(report)
|
|
30
|
+
|
|
31
|
+
return EXIT_TOOL if unverifiable_scope?(report)
|
|
32
|
+
|
|
12
33
|
return EXIT_TOOL if mostly_errored?(report)
|
|
13
34
|
|
|
14
35
|
return EXIT_TOOL if incomplete_under_threshold?(report)
|
|
@@ -16,6 +37,50 @@ module Saboteur
|
|
|
16
37
|
failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
|
|
17
38
|
end
|
|
18
39
|
|
|
40
|
+
# A spent --max-duration budget exits 4 on its own, before the gates:
|
|
41
|
+
# the run is partial by design, and --max-duration-ok opts back into
|
|
42
|
+
# the plain verdict.
|
|
43
|
+
def budget_exhausted?(report)
|
|
44
|
+
report.early_stop&.reason == "max_duration" && !config["max_duration_ok"]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# A run stopped before any mutant was tested has no verdict to fall
|
|
48
|
+
# back on: --max-duration-ok must not turn it green. Unverified
|
|
49
|
+
# results are not tested verdicts, so an unverified-only stop has no
|
|
50
|
+
# score either (#296). Exit 3, like the other incomplete shapes.
|
|
51
|
+
def untested_partial?(report)
|
|
52
|
+
return false unless report.early_stop && report.score_breakdown.tested.zero?
|
|
53
|
+
|
|
54
|
+
@err.puts("saboteur: the run stopped before any mutant was tested, so there is no score to judge")
|
|
55
|
+
true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A run that verified none of the change under review - every
|
|
59
|
+
# subject skipped (refinements, no coverage, ..., #574) or every
|
|
60
|
+
# mutant unreachable (a suite re-loading mutated files, #597) -
|
|
61
|
+
# must not exit 0 with score n/a and green-light an untested change.
|
|
62
|
+
def unverifiable_scope?(report)
|
|
63
|
+
if unverifiable_review?(report)
|
|
64
|
+
report.skipped_subjects.each do |subject, reason|
|
|
65
|
+
@err.puts("saboteur: #{subject.id} was skipped (#{reason}), so the change under test was not verified")
|
|
66
|
+
end
|
|
67
|
+
return true
|
|
68
|
+
end
|
|
69
|
+
return false unless unverifiable_unreachable?(report)
|
|
70
|
+
|
|
71
|
+
@err.puts("saboteur: all mutants ran unreachable, so the change under test was not verified")
|
|
72
|
+
true
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def unverifiable_review?(report)
|
|
76
|
+
report.review? && report.subjects.any? &&
|
|
77
|
+
report.subjects.all? { |subject| report.skipped_subjects.key?(subject) }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def unverifiable_unreachable?(report)
|
|
81
|
+
report.executed.any? && report.executed.all? { |result| result.status == :unreachable }
|
|
82
|
+
end
|
|
83
|
+
|
|
19
84
|
def gates = (@gates ||= Gates.new(config))
|
|
20
85
|
|
|
21
86
|
def mostly_errored?(report)
|
|
@@ -33,9 +98,11 @@ module Saboteur
|
|
|
33
98
|
# incomplete run that mostly errored keeps its distinct cause, so the
|
|
34
99
|
# harness rule means the same thing in both modes.
|
|
35
100
|
def agent_exit_cause(report, payload)
|
|
101
|
+
return "budgetExhausted" if budget_exhausted?(report)
|
|
102
|
+
|
|
36
103
|
case payload[:verdict]
|
|
37
104
|
when "clean" then "ok"
|
|
38
|
-
when "survivors" then report.early_stop ? "failFast" : "survivors"
|
|
105
|
+
when "survivors" then report.early_stop&.reason == "fail_fast" ? "failFast" : "survivors"
|
|
39
106
|
else report.majority_errored? ? "errorMajority" : "incomplete"
|
|
40
107
|
end
|
|
41
108
|
end
|
|
@@ -45,22 +112,31 @@ module Saboteur
|
|
|
45
112
|
def exit_cause(report, code)
|
|
46
113
|
case code
|
|
47
114
|
when EXIT_OK then "ok"
|
|
115
|
+
when EXIT_BUDGET then "budgetExhausted"
|
|
48
116
|
when EXIT_SURVIVORS then survivors_cause(report)
|
|
49
|
-
else
|
|
117
|
+
else
|
|
118
|
+
if report.majority_errored?
|
|
119
|
+
"errorMajority"
|
|
120
|
+
elsif unverifiable_review?(report) || unverifiable_unreachable?(report) ||
|
|
121
|
+
(report.early_stop && report.score_breakdown.tested.zero?)
|
|
122
|
+
"incomplete"
|
|
123
|
+
else
|
|
124
|
+
"incompleteUnderThreshold"
|
|
125
|
+
end
|
|
50
126
|
end
|
|
51
127
|
end
|
|
52
128
|
|
|
53
129
|
# A run stopped by --fail-fast always exits 1: it stopped on a survivor, and
|
|
54
130
|
# its score covers only the subjects that ran, so no threshold can pass it.
|
|
55
131
|
def survivors_cause(report)
|
|
56
|
-
return "failFast" if report.early_stop
|
|
132
|
+
return "failFast" if report.early_stop&.reason == "fail_fast"
|
|
57
133
|
|
|
58
134
|
gates.cause || (config["threshold"].nil? ? "survivors" : "belowThreshold")
|
|
59
135
|
end
|
|
60
136
|
|
|
61
137
|
# With --threshold the score decides; without it any unignored survivor does.
|
|
62
138
|
def failed_quality_gate?(report)
|
|
63
|
-
return true if report.early_stop
|
|
139
|
+
return true if report.early_stop&.reason == "fail_fast"
|
|
64
140
|
|
|
65
141
|
threshold = config["threshold"]
|
|
66
142
|
return report.unignored_survivors.any? if threshold.nil?
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The option map `saboteur help` prints per command. run, pr and audit
|
|
7
|
+
# share the run parser: the presets take every run option.
|
|
8
|
+
module HelpOptions
|
|
9
|
+
def option_lines
|
|
10
|
+
run = parser.summarize
|
|
11
|
+
{ "run" => run, "pr" => run, "audit" => run, "doctor" => doctor_parser.summarize,
|
|
12
|
+
"baseline" => baseline_parser.summarize, "badge" => badge_parser.summarize,
|
|
13
|
+
"trend" => trend_parser.summarize }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../reporter/history_reader"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class CLI
|
|
8
|
+
# `saboteur badge` and `saboteur trend` (#474): a local README badge and
|
|
9
|
+
# a trend read-out over the opt-in score history, no hosted service.
|
|
10
|
+
module HistoryCommand
|
|
11
|
+
DEFAULT_HIGH = 80.0
|
|
12
|
+
DEFAULT_LOW = 60.0
|
|
13
|
+
|
|
14
|
+
def badge_parser
|
|
15
|
+
OptionParser.new do |opts|
|
|
16
|
+
opts.banner = "Usage: saboteur badge [--history PATH] [--output PATH] [--mark-partial] " \
|
|
17
|
+
"[--high PERCENT] [--low PERCENT]"
|
|
18
|
+
opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
|
|
19
|
+
opts.on("--output PATH", "Badge JSON destination (default: stdout)") { |v| @output = v }
|
|
20
|
+
opts.on("--mark-partial", "Badge an incomplete run, labeled as partial") { @mark_partial = true }
|
|
21
|
+
opts.on("--high PERCENT", Float, "brightgreen at or above (default #{DEFAULT_HIGH.to_i})") { |v| @high = v }
|
|
22
|
+
opts.on("--low PERCENT", Float, "yellow at or above (default #{DEFAULT_LOW.to_i})") { |v| @low = v }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def trend_parser
|
|
27
|
+
OptionParser.new do |opts|
|
|
28
|
+
opts.banner = "Usage: saboteur trend [--history PATH]"
|
|
29
|
+
opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def history_command(command) = command == "badge" ? badge : trend
|
|
36
|
+
|
|
37
|
+
def badge
|
|
38
|
+
badge_parser.parse!(@argv)
|
|
39
|
+
raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
|
|
40
|
+
|
|
41
|
+
write_badge(badge_payload(read_history.last))
|
|
42
|
+
EXIT_OK
|
|
43
|
+
rescue ConfigError, Error, SystemCallError => e
|
|
44
|
+
history_failure(e)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def trend
|
|
48
|
+
trend_parser.parse!(@argv)
|
|
49
|
+
raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
|
|
50
|
+
|
|
51
|
+
print_trend(latest_pair)
|
|
52
|
+
EXIT_OK
|
|
53
|
+
rescue ConfigError, Error, SystemCallError => e
|
|
54
|
+
history_failure(e)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def history_failure(error)
|
|
58
|
+
@err.puts("saboteur: #{error.message}")
|
|
59
|
+
error.is_a?(ConfigError) ? EXIT_USAGE : EXIT_TOOL
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def latest_pair
|
|
63
|
+
records = read_history
|
|
64
|
+
current = records.last
|
|
65
|
+
[records[0...-1].reverse.find { |record| Reporter::HistoryReader.comparable?(record, current) }, current]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def read_history
|
|
69
|
+
records = Reporter::HistoryReader.read(history_path)
|
|
70
|
+
if records.empty?
|
|
71
|
+
raise Error,
|
|
72
|
+
"score history is empty - record it first by running saboteur with c.score_history set"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
records
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def history_path
|
|
79
|
+
@history_path ||= config.path("score_history")
|
|
80
|
+
@history_path or raise UsageError, "no score history: pass --history PATH or set c.score_history"
|
|
81
|
+
rescue ConfigError
|
|
82
|
+
raise UsageError, "no score history: pass --history PATH or set c.score_history"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def badge_payload(record)
|
|
86
|
+
partial = record["incomplete"].any?
|
|
87
|
+
if partial && !@mark_partial
|
|
88
|
+
raise Error, "the latest score history record is an incomplete run " \
|
|
89
|
+
"(#{record['incomplete'].map(&:inspect).join(', ')}) - " \
|
|
90
|
+
"badge evidence needs a complete run, or pass --mark-partial to label it"
|
|
91
|
+
end
|
|
92
|
+
score = record["coveredScore"]
|
|
93
|
+
raise Error, "the latest score history record has no tested mutants" if score.nil?
|
|
94
|
+
|
|
95
|
+
{ schemaVersion: 1, label: partial ? "mutation score (partial)" : "mutation score",
|
|
96
|
+
message: format("%.1f%%", score * 100), color: badge_color(score) }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def write_badge(payload)
|
|
100
|
+
json = "#{JSON.generate(payload)}\n"
|
|
101
|
+
return @out.print(json) unless @output
|
|
102
|
+
|
|
103
|
+
File.write(@output, json)
|
|
104
|
+
@out.puts(@output)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def badge_color(score)
|
|
108
|
+
percent = score * 100
|
|
109
|
+
return "brightgreen" if percent >= (@high || DEFAULT_HIGH)
|
|
110
|
+
return "yellow" if percent >= (@low || DEFAULT_LOW)
|
|
111
|
+
|
|
112
|
+
"red"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def print_trend((previous, current))
|
|
116
|
+
([previous].compact + [current]).each { |record| @out.puts(trend_line(record)) }
|
|
117
|
+
unless previous && previous["coveredScore"] && current["coveredScore"]
|
|
118
|
+
return @out.puts("score history: no comparable earlier record (different selection/inventory, " \
|
|
119
|
+
"operator policy or an incomplete run)")
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
delta = (current["coveredScore"] - previous["coveredScore"]) * 100
|
|
123
|
+
@out.puts(format("covered score change %<delta>+.2f percentage points since %<since>s " \
|
|
124
|
+
"(same configured selection)", delta: delta, since: previous["date"]))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def trend_line(record)
|
|
128
|
+
suffix = record["incomplete"].empty? ? "" : " (incomplete)"
|
|
129
|
+
score = record["coveredScore"]
|
|
130
|
+
"#{record['date']} #{record['commit']&.slice(0, 8) || '-'} " \
|
|
131
|
+
"#{score.nil? ? 'no tested mutants' : format('%.1f%%', score * 100)}#{suffix}"
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# Opinionated bundles (#244): pr is the fast changed-lines loop (one
|
|
7
|
+
# baseline check, a 10-minute budget that fails an incomplete run),
|
|
8
|
+
# audit is the whole project with both reports. Preset flags go first,
|
|
9
|
+
# so an explicit flag after them wins on the usual last-flag rule.
|
|
10
|
+
module Presets
|
|
11
|
+
PRESETS = {
|
|
12
|
+
"pr" => %w[--baseline-passes 1 --max-duration 10m],
|
|
13
|
+
"audit" => %w[--all --operators all --baseline-passes 2 --json .saboteur/audit-report.json
|
|
14
|
+
--html .saboteur/audit-report.html]
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def preset_or_usage_error(command)
|
|
20
|
+
PRESETS.key?(command) ? preset(command) : usage_error
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def preset(name)
|
|
24
|
+
flags = PRESETS.fetch(name).dup
|
|
25
|
+
if name == "pr"
|
|
26
|
+
ref = @argv.first.nil? || @argv.first.start_with?("-") ? "main" : @argv.shift
|
|
27
|
+
flags += ["--since", ref]
|
|
28
|
+
end
|
|
29
|
+
# An explicit scope beats the injected whole-suite --all, which
|
|
30
|
+
# would otherwise reject the combination: named tests, and the
|
|
31
|
+
# deprecated inference alias (#245).
|
|
32
|
+
explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
|
|
33
|
+
flags.delete("--all") if explicit_scope
|
|
34
|
+
@err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
|
|
35
|
+
@argv = flags + @argv
|
|
36
|
+
run_command
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The run's clock and repeatability switches: seed, baseline check count
|
|
7
|
+
# (#55) and the wall-clock budget (#245). Grouped so the option table in
|
|
8
|
+
# the CLI stays under the house size caps.
|
|
9
|
+
module RunBudgetOptions
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def add_run_budget_options(opts)
|
|
13
|
+
opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
|
|
14
|
+
set(:seed, v)
|
|
15
|
+
end
|
|
16
|
+
opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
|
|
17
|
+
opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent") do |v|
|
|
18
|
+
set(:max_duration, parse_duration(v))
|
|
19
|
+
end
|
|
20
|
+
opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|
|
|
21
|
+
set(:max_duration_ok, v)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# "10m" -> 600.0; bare numbers are seconds. A bad value is a usage error.
|
|
26
|
+
def parse_duration(value)
|
|
27
|
+
match = /\A(?<amount>\d+(?:\.\d+)?)(?<unit>[smh])?\z/.match(value.to_s.strip)
|
|
28
|
+
unless match
|
|
29
|
+
raise ConfigError,
|
|
30
|
+
"invalid --max-duration #{value.inspect}: use seconds or an s/m/h suffix, e.g. 10m"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
match[:amount].to_f * { "s" => 1, "m" => 60, "h" => 3600, nil => 1 }.fetch(match[:unit])
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -10,9 +10,10 @@ module Saboteur
|
|
|
10
10
|
module RunReporting
|
|
11
11
|
private
|
|
12
12
|
|
|
13
|
-
def text_run(report)
|
|
13
|
+
def text_run(report, engine)
|
|
14
14
|
@err.puts
|
|
15
|
-
Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto
|
|
15
|
+
Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto,
|
|
16
|
+
replay_command: survivor_replay_command(report, engine)).call(report)
|
|
16
17
|
Reporter::Replay.new(@out).call(report) if options[:id]
|
|
17
18
|
code = exit_code(report)
|
|
18
19
|
# The JSON carries the exit cause an outer harness trusts, so it is
|
|
@@ -50,7 +51,65 @@ module Saboteur
|
|
|
50
51
|
validate_history
|
|
51
52
|
arguments = { err: @err }
|
|
52
53
|
arguments[:digest] = options[:id] if options[:id]
|
|
53
|
-
|
|
54
|
+
return agent_run(arguments) if agent?
|
|
55
|
+
|
|
56
|
+
# Text mode only: the engine prints a per-subject marks line when
|
|
57
|
+
# --verbose is on (#262); --agent output stays as it is.
|
|
58
|
+
engine = Engine.new(config, **arguments, verbose: @verbose || false)
|
|
59
|
+
text_run(engine.call, engine)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The full replay command for a survivor digest: this run's own
|
|
63
|
+
# selection options, assembled by the same code that writes the agent
|
|
64
|
+
# follow-ups, so the user never reassembles them by hand (#261).
|
|
65
|
+
# Computed lazily: only a run with survivors asks for it.
|
|
66
|
+
def survivor_replay_command(report, engine)
|
|
67
|
+
lambda do |digest|
|
|
68
|
+
tree = WorkingTree.new(config.root, ignored: config.output_paths).state
|
|
69
|
+
Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Writes the JSON report and returns its path, or nil when none is configured.
|
|
74
|
+
#
|
|
75
|
+
# An explicitly requested JSON report (--json, c.json_report) is the
|
|
76
|
+
# completion evidence an outer harness trusts: a failed write fails
|
|
77
|
+
# the run (exit 3), because a warning with exit 0 could leave a stale
|
|
78
|
+
# or partial file at the path and present old results as this run's.
|
|
79
|
+
# The default .saboteur/report.json is best-effort like the HTML
|
|
80
|
+
# report and the score history (#296).
|
|
81
|
+
def write_json(report, cause)
|
|
82
|
+
path = config.path("json_report")
|
|
83
|
+
return unless path
|
|
84
|
+
|
|
85
|
+
Reporter::Stryker.new(path).call(report, exit_cause: cause)
|
|
86
|
+
path
|
|
87
|
+
rescue SystemCallError, IOError => e
|
|
88
|
+
if config.explicit?("json_report")
|
|
89
|
+
raise Error, "could not write the JSON report to #{path}: #{e.message} - " \
|
|
90
|
+
"any file at that path may be stale or incomplete"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
warn_report_write_failure("JSON report", path, e)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def write_html(report)
|
|
97
|
+
path = config.path("html_report")
|
|
98
|
+
return unless path
|
|
99
|
+
|
|
100
|
+
Reporter::Html.new(path).call(report)
|
|
101
|
+
path
|
|
102
|
+
rescue SystemCallError, IOError => e
|
|
103
|
+
warn_report_write_failure("HTML report", path, e)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# An auxiliary report artifact that cannot be written - the tree is
|
|
107
|
+
# read-only inside a sandboxed child, as in the dogfood's nested runs
|
|
108
|
+
# (#296) - is a warning, not a run failure: the exit code belongs to
|
|
109
|
+
# the results. The JSON report is not auxiliary; see write_json.
|
|
110
|
+
def warn_report_write_failure(what, path, error)
|
|
111
|
+
@err.puts "saboteur: could not write the #{what} to #{path} (#{error.message}); continuing without it"
|
|
112
|
+
nil
|
|
54
113
|
end
|
|
55
114
|
|
|
56
115
|
def validate_history
|
|
@@ -67,8 +126,18 @@ module Saboteur
|
|
|
67
126
|
|
|
68
127
|
tree = WorkingTree.new(config.root, ignored: config.output_paths).state
|
|
69
128
|
scope = config.selection_settings.except("json_report", "html_report", "score_history", "cache_dir")
|
|
70
|
-
|
|
71
|
-
|
|
129
|
+
# Recording is an auxiliary artifact: when the file cannot be
|
|
130
|
+
# written - read-only tree inside a sandboxed child (#296) - warn
|
|
131
|
+
# and let the run exit by its results instead of failing as a tool
|
|
132
|
+
# error. Only the write's IO failures are soft: validation,
|
|
133
|
+
# integrity, record-size and lock failures (Saboteur::Error) stay
|
|
134
|
+
# hard, because they say the history is unusable, not unwritable.
|
|
135
|
+
message = begin
|
|
136
|
+
Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
|
|
137
|
+
rescue SystemCallError, IOError => e
|
|
138
|
+
warn_report_write_failure("score history", path, e)
|
|
139
|
+
end
|
|
140
|
+
@err.puts("saboteur: #{message}") if message
|
|
72
141
|
end
|
|
73
142
|
end
|
|
74
143
|
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The run's subject-selection flags: operator sets, single-mutant replay,
|
|
7
|
+
# --since scoping, --dry-run, the flaky-check confirmation rerun (#476)
|
|
8
|
+
# and explicit --subject filters. Grouped so the CLI stays under the
|
|
9
|
+
# house size caps.
|
|
10
|
+
module SelectionOptions
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
def define_selection_options(opts)
|
|
14
|
+
opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
|
|
15
|
+
set(:operators, v)
|
|
16
|
+
end
|
|
17
|
+
opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
|
|
18
|
+
set(:exclude_operators, v)
|
|
19
|
+
end
|
|
20
|
+
opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
|
|
21
|
+
opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
|
|
22
|
+
opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
|
|
23
|
+
opts.on("--shard K/N", "Run one of N deterministic candidate shards (for example 1/4)") { |v| set(:shard, v) }
|
|
24
|
+
opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
|
|
25
|
+
flaky_desc = "Confirm a fresh verdict (survivors, decisive) with one reordered rerun"
|
|
26
|
+
opts.on("--flaky-check MODE", %w[off survivors decisive], flaky_desc) { |v| set(:flaky_check, v) }
|
|
27
|
+
opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# Validates the deterministic candidate-shard selection before the target boots.
|
|
7
|
+
module ShardOptions
|
|
8
|
+
private
|
|
9
|
+
|
|
10
|
+
def validate_shard_options
|
|
11
|
+
raise UsageError, "--id cannot be combined with --shard" if options[:id] && config["shard"]
|
|
12
|
+
|
|
13
|
+
Engine::Shard.parse(config["shard"]) if config["shard"]
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|