saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The effective source set a run will mutate, refused when it is empty.
|
|
7
|
+
module SourceSet
|
|
8
|
+
private
|
|
9
|
+
|
|
10
|
+
# The Rails-shaped footgun (#447): the default include targets plain
|
|
11
|
+
# gems, so a first run on an app/ tree selects nothing and would exit 0
|
|
12
|
+
# over 0 subjects - a green gate that mutates nothing, every CI run.
|
|
13
|
+
# Refuse like a typo'd source: exit 2, name the stale pattern and the
|
|
14
|
+
# way out. --dry-run still previews: it gates nothing.
|
|
15
|
+
def refuse_empty_source_set
|
|
16
|
+
return unless config.included_files.empty?
|
|
17
|
+
|
|
18
|
+
raise UsageError, empty_source_set_message
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# A --since ref whose resolved range diffs empty yields 0 subjects
|
|
22
|
+
# and a green run over nothing: `saboteur pr <merge_sha>` on a
|
|
23
|
+
# checkout AT the merge diffs the merge against itself (#643).
|
|
24
|
+
# Refuse like a typo'd ref: exit 2, name the base and the range.
|
|
25
|
+
def refuse_empty_since_range
|
|
26
|
+
ref = config["since"]
|
|
27
|
+
return unless ref
|
|
28
|
+
|
|
29
|
+
paths = Selection::GitDiff.changed_paths(ref, root: config.root)
|
|
30
|
+
return unless paths.values.all?(&:empty?)
|
|
31
|
+
|
|
32
|
+
base = Selection::GitDiff.merge_base(ref, root: config.root)
|
|
33
|
+
raise UsageError, "--since #{ref}: the resolved range #{base}..the working tree has no changes - " \
|
|
34
|
+
"the run would pass with 0 subjects. A checkout AT the ref (the CI merge-sha " \
|
|
35
|
+
"checkout) diffs it against itself: fetch the merge's parent, or point --since " \
|
|
36
|
+
"at the base of the change under test"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def empty_source_set_message
|
|
40
|
+
include_patterns = config["include"]
|
|
41
|
+
matched = include_patterns.flat_map { |pattern| Dir.glob(pattern, base: config.root) }
|
|
42
|
+
.select { |file| file.end_with?(".rb") }
|
|
43
|
+
cause = if matched.empty?
|
|
44
|
+
"include: #{include_patterns.join(', ')} selects no Ruby file"
|
|
45
|
+
else
|
|
46
|
+
"exclude: #{config['exclude'].join(', ')} removes every included file"
|
|
47
|
+
end
|
|
48
|
+
"#{cause} - the run would pass with 0 subjects. Point include at your code " \
|
|
49
|
+
"(Rails apps: bundle exec rails generate saboteur:install writes the config) or fix the patterns"
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class CLI
|
|
8
|
+
# saboteur survivors <report.json> (#475): a read-only, line-oriented
|
|
9
|
+
# terminal browser for the Stryker JSON report. It never boots the
|
|
10
|
+
# project or reruns tests; it lists survivors, filters them, and for
|
|
11
|
+
# one digest prints the diff, the exact replay command and the
|
|
12
|
+
# ignore-file template. It never edits the ignore file.
|
|
13
|
+
module SurvivorsCommand
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def survivors_command
|
|
17
|
+
survivors_command_inner
|
|
18
|
+
rescue UsageError => e
|
|
19
|
+
@err.puts("saboteur: #{e.message}")
|
|
20
|
+
EXIT_USAGE
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def survivors_command_inner
|
|
24
|
+
parser = OptionParser.new do |opts|
|
|
25
|
+
opts.on("--ranked", "Review-priority order: the report's ranked findings first")
|
|
26
|
+
opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
|
|
27
|
+
opts.on("--operator NAME", "Only survivors of operator NAME")
|
|
28
|
+
opts.on("--triage LABEL", "Only survivors classified LABEL (for example missing_test)")
|
|
29
|
+
opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
|
|
30
|
+
end
|
|
31
|
+
options = {}
|
|
32
|
+
args = parser.parse!(@argv, into: options)
|
|
33
|
+
path = args.shift or raise UsageError,
|
|
34
|
+
"survivors expects a report path: saboteur survivors .saboteur/report.json"
|
|
35
|
+
raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
|
|
36
|
+
|
|
37
|
+
report = load_survivor_report(path)
|
|
38
|
+
browser = SurvivorsBrowser.new(report)
|
|
39
|
+
if options[:show]
|
|
40
|
+
@out.puts(browser.show(options[:show]))
|
|
41
|
+
else
|
|
42
|
+
@out.puts(browser.list(**options.slice(:ranked, :file, :operator, :triage)))
|
|
43
|
+
end
|
|
44
|
+
EXIT_OK
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def load_survivor_report(path)
|
|
48
|
+
raise UsageError, "report not found: #{path}" unless File.file?(path)
|
|
49
|
+
|
|
50
|
+
JSON.parse(File.read(path))
|
|
51
|
+
rescue JSON::ParserError => e
|
|
52
|
+
raise UsageError, "#{path} is not a readable saboteur report: #{e.message}"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Reads the report's files section (the complete mutant list), not the
|
|
57
|
+
# capped findings, so survivors beyond the findings limit still list.
|
|
58
|
+
class SurvivorsBrowser
|
|
59
|
+
def initialize(report)
|
|
60
|
+
@report = report
|
|
61
|
+
@findings = report.dig("saboteur", "findings") || {}
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def list(ranked: false, file: nil, operator: nil, triage: nil)
|
|
65
|
+
survivors = filtered(file, operator, triage)
|
|
66
|
+
survivors = ranked_order(survivors) if ranked
|
|
67
|
+
return "no survivors#{filters_note(file, operator, triage)}" if survivors.empty?
|
|
68
|
+
|
|
69
|
+
[header(survivors.size, file, operator, triage), *rows(survivors)]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def show(digest)
|
|
73
|
+
survivor = all_survivors.find { |s| s[:digest].start_with?(digest) } ||
|
|
74
|
+
raise(UsageError, "no survivor with digest #{digest} in this report")
|
|
75
|
+
finding = findings_by_digest[survivor[:digest]] || {}
|
|
76
|
+
subject = finding["subject"]
|
|
77
|
+
[
|
|
78
|
+
"#{survivor[:path]}:#{survivor[:line]} #{survivor[:operator]} #{survivor[:digest]}",
|
|
79
|
+
"",
|
|
80
|
+
"- #{finding['original'] || original_line(survivor)}",
|
|
81
|
+
"+ #{finding['replacement'] || survivor[:replacement]}",
|
|
82
|
+
"",
|
|
83
|
+
"Replay exactly this mutant:",
|
|
84
|
+
" bundle exec saboteur run --id #{survivor[:digest]}",
|
|
85
|
+
"",
|
|
86
|
+
"Ignore-file entry (config/saboteur_ignore.rb):",
|
|
87
|
+
*ignore_entry(survivor, subject)
|
|
88
|
+
].join("\n")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def filtered(file, operator, triage)
|
|
94
|
+
survivors = all_survivors
|
|
95
|
+
survivors = survivors.select { |s| s[:path].include?(file) } if file
|
|
96
|
+
survivors = survivors.select { |s| s[:operator] == operator } if operator
|
|
97
|
+
triage ? survivors.select { |s| s[:classification] == triage } : survivors
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def header(size, file, operator, triage)
|
|
101
|
+
text = "#{size} survivor#{'s' unless size == 1}#{filters_note(file, operator, triage)}"
|
|
102
|
+
truncated = @findings["survivorsTruncated"].to_i
|
|
103
|
+
return text unless truncated.positive?
|
|
104
|
+
|
|
105
|
+
"#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def rows(survivors)
|
|
109
|
+
survivors.each_with_index.map do |s, i|
|
|
110
|
+
format("%3<i>d. %<path>s:%<line>d %-28<operator>s %<digest>s %<classification>s",
|
|
111
|
+
i: i + 1, path: s[:path], line: s[:line], operator: s[:operator], digest: s[:digest],
|
|
112
|
+
classification: s[:classification] || "-")
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def ignore_entry(survivor, subject)
|
|
117
|
+
note = subject ? subject.inspect : '"Klass#meth" # TODO: capped findings omit this digest - fill it in'
|
|
118
|
+
[%( i.mutation "#{survivor[:digest]}",), " subject: #{note},", %( reason: "...")]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def all_survivors
|
|
122
|
+
@all_survivors ||= begin
|
|
123
|
+
survivors = @report.fetch("files", {}).flat_map do |path, entry|
|
|
124
|
+
entry.fetch("mutants", []).select { |m| m["status"] == "Survived" }.map do |m|
|
|
125
|
+
{ path: path, line: m.dig("location", "start", "line").to_i, operator: m["mutatorName"],
|
|
126
|
+
digest: m["id"], replacement: m["replacement"], classification: m["classification"],
|
|
127
|
+
covered: m["coveredBy"] || [], source: entry["source"] }
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
survivors.sort_by { |s| [s[:path], s[:line], s[:digest]] }
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def ranked_order(survivors)
|
|
135
|
+
rank = @findings.fetch("survivors", []).each_with_index.to_h { |f, i| [f["digest"], i] }
|
|
136
|
+
survivors.sort_by { |s| [rank[s[:digest]] || Float::INFINITY, -s[:covered].size, s[:path], s[:line], s[:digest]] }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def findings_by_digest
|
|
140
|
+
@findings_by_digest ||=
|
|
141
|
+
(@findings.fetch("survivors", []) + @findings.fetch("accepted", [])).to_h { |f| [f["digest"], f] }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def original_line(survivor)
|
|
145
|
+
survivor[:source].to_s.lines[survivor[:line] - 1].to_s.strip
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def filters_note(file, operator, triage)
|
|
149
|
+
parts = []
|
|
150
|
+
parts << "file~#{file}" if file
|
|
151
|
+
parts << "operator=#{operator}" if operator
|
|
152
|
+
parts << "triage=#{triage}" if triage
|
|
153
|
+
parts.empty? ? "" : " (#{parts.join(', ')})"
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "optparse"
|
|
5
5
|
require_relative "../saboteur"
|
|
6
|
+
require_relative "crash_report"
|
|
6
7
|
require_relative "doctor"
|
|
7
8
|
require_relative "help"
|
|
8
9
|
require_relative "reporter/replay"
|
|
@@ -10,14 +11,30 @@ require_relative "baseline/command"
|
|
|
10
11
|
require_relative "cli/exit_status"
|
|
11
12
|
require_relative "preview"
|
|
12
13
|
require_relative "cli/run_reporting"
|
|
14
|
+
require_relative "cli/survivors_command"
|
|
15
|
+
require_relative "cli/source_set"
|
|
16
|
+
require_relative "cli/run_budget_options"
|
|
17
|
+
require_relative "cli/shard_options"
|
|
18
|
+
require_relative "cli/selection_options"
|
|
19
|
+
require_relative "cli/presets"
|
|
20
|
+
require_relative "cli/help_options"
|
|
21
|
+
require_relative "cli/history_command"
|
|
13
22
|
|
|
14
23
|
module Saboteur
|
|
15
24
|
class CLI
|
|
16
25
|
include Baseline::Command
|
|
17
26
|
include ExitStatus
|
|
18
27
|
include RunReporting
|
|
19
|
-
|
|
20
|
-
|
|
28
|
+
include SurvivorsCommand
|
|
29
|
+
include SourceSet
|
|
30
|
+
include RunBudgetOptions
|
|
31
|
+
include ShardOptions
|
|
32
|
+
include SelectionOptions
|
|
33
|
+
include Presets
|
|
34
|
+
include HistoryCommand
|
|
35
|
+
include HelpOptions
|
|
36
|
+
|
|
37
|
+
USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
|
|
21
38
|
"[--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N] [--id DIGEST]
|
|
22
39
|
" \
|
|
23
40
|
"Run 'saboteur help' for every command, option, config key and exit code."
|
|
@@ -26,10 +43,18 @@ module Saboteur
|
|
|
26
43
|
EXIT_SURVIVORS = 1
|
|
27
44
|
EXIT_USAGE = 2
|
|
28
45
|
EXIT_TOOL = 3
|
|
46
|
+
EXIT_BUDGET = 4
|
|
29
47
|
class UsageError < ConfigError; end
|
|
30
48
|
|
|
31
49
|
attr_reader :options
|
|
32
50
|
|
|
51
|
+
# Found before parsing, so any error, even a bad option that comes first, still
|
|
52
|
+
# answers an agent with JSON. Class-level: the executable's fallback reads raw arguments.
|
|
53
|
+
def self.agent_requested?(argv)
|
|
54
|
+
argv.include?("--agent") || argv.include?("--format=agent") ||
|
|
55
|
+
argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
|
|
56
|
+
end
|
|
57
|
+
|
|
33
58
|
def initialize(argv, out: $stdout, err: $stderr)
|
|
34
59
|
@argv = argv.dup
|
|
35
60
|
@out = out
|
|
@@ -39,18 +64,21 @@ module Saboteur
|
|
|
39
64
|
end
|
|
40
65
|
|
|
41
66
|
def call
|
|
42
|
-
|
|
67
|
+
command = @argv.shift
|
|
68
|
+
case command
|
|
43
69
|
when "run" then run_command
|
|
44
70
|
when "baseline" then baseline
|
|
45
|
-
when "
|
|
71
|
+
when "survivors" then survivors_command
|
|
72
|
+
when "doctor" then doctor_command
|
|
73
|
+
when "badge", "trend" then history_entry(command)
|
|
46
74
|
when "version", "--version" then @out.puts(VERSION) || EXIT_OK
|
|
47
|
-
when "help" then show_help(@argv.shift)
|
|
75
|
+
when "help", "--help", "-h" then show_help(@argv.shift)
|
|
48
76
|
when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
|
|
49
|
-
|
|
50
|
-
else usage_error
|
|
77
|
+
else preset_or_usage_error(command)
|
|
51
78
|
end
|
|
52
79
|
rescue OptionParser::ParseError => e
|
|
53
80
|
usage_error(e.message)
|
|
81
|
+
rescue StandardError => e then internal_error(e)
|
|
54
82
|
end
|
|
55
83
|
|
|
56
84
|
def config
|
|
@@ -68,11 +96,6 @@ module Saboteur
|
|
|
68
96
|
end
|
|
69
97
|
end
|
|
70
98
|
|
|
71
|
-
# Each command's own option lines, as printed under OPTIONS in `saboteur help`.
|
|
72
|
-
def option_lines
|
|
73
|
-
{ "run" => parser.summarize, "doctor" => doctor_parser.summarize, "baseline" => baseline_parser.summarize }
|
|
74
|
-
end
|
|
75
|
-
|
|
76
99
|
def doctor_parser
|
|
77
100
|
OptionParser.new do |opts|
|
|
78
101
|
opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N] [--suite] [--test PATH]"
|
|
@@ -87,13 +110,13 @@ module Saboteur
|
|
|
87
110
|
|
|
88
111
|
private
|
|
89
112
|
|
|
90
|
-
def run_command
|
|
91
|
-
help_requested? ? show_help("run") : run
|
|
92
|
-
end
|
|
113
|
+
def run_command = help_requested? ? show_help("run") : run
|
|
93
114
|
|
|
94
|
-
def help_requested?
|
|
95
|
-
|
|
96
|
-
|
|
115
|
+
def doctor_command = help_requested? ? show_help("doctor") : doctor
|
|
116
|
+
|
|
117
|
+
def history_entry(command) = help_requested? ? show_help(command) : history_command(command)
|
|
118
|
+
|
|
119
|
+
def help_requested? = @argv.intersect?(%w[--help -h])
|
|
97
120
|
|
|
98
121
|
def show_help(command_name)
|
|
99
122
|
help = Help.new(option_lines: option_lines)
|
|
@@ -110,16 +133,9 @@ module Saboteur
|
|
|
110
133
|
agent_failure(message || USAGE, EXIT_USAGE)
|
|
111
134
|
end
|
|
112
135
|
|
|
113
|
-
def agent?
|
|
114
|
-
@format == "agent"
|
|
115
|
-
end
|
|
136
|
+
def agent? = @format == "agent"
|
|
116
137
|
|
|
117
|
-
|
|
118
|
-
# answers an agent with JSON.
|
|
119
|
-
def agent_requested?
|
|
120
|
-
@argv.include?("--agent") || @argv.include?("--format=agent") ||
|
|
121
|
-
@argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
|
|
122
|
-
end
|
|
138
|
+
def agent_requested? = self.class.agent_requested?(@argv)
|
|
123
139
|
|
|
124
140
|
# In agent mode the failure is also on stdout, where the agent reads its answer.
|
|
125
141
|
def agent_failure(message, exit_code)
|
|
@@ -129,12 +145,17 @@ module Saboteur
|
|
|
129
145
|
|
|
130
146
|
def run
|
|
131
147
|
parser.parse!(@argv)
|
|
148
|
+
reject_test_selection_conflicts
|
|
132
149
|
use_explicit_sources
|
|
133
150
|
validate_run_options
|
|
134
151
|
warn_dirty_tree
|
|
135
152
|
|
|
153
|
+
config.deprecation_warnings.each { |warning| @err.puts("saboteur: #{warning}") }
|
|
154
|
+
|
|
136
155
|
return preview if options[:list_operators] || options[:dry_run]
|
|
137
156
|
|
|
157
|
+
refuse_empty_source_set
|
|
158
|
+
refuse_empty_since_range
|
|
138
159
|
execute_run
|
|
139
160
|
rescue ConfigError => e
|
|
140
161
|
@err.puts("saboteur: #{e.message}")
|
|
@@ -146,28 +167,60 @@ module Saboteur
|
|
|
146
167
|
|
|
147
168
|
# Mutants run your code, and a mutant of code that writes or deletes files
|
|
148
169
|
# can change the tree. Say so up front when uncommitted work is at stake.
|
|
170
|
+
# The scope the warning names is the configured one: positional source
|
|
171
|
+
# operands narrow the run after the warning, so the include/exclude they
|
|
172
|
+
# imply stay out of its config (#309, #503 review).
|
|
149
173
|
def warn_dirty_tree
|
|
150
|
-
|
|
174
|
+
warning = @sources ? Config.load(path: options[:config], overrides: overrides.except(:include, :exclude)) : config
|
|
175
|
+
DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Contradictory selection flags are a usage error before any file or
|
|
179
|
+
# repository check: --test wins over inference by precedence, but only
|
|
180
|
+
# one of them may be named. Config-file combinations are not errors:
|
|
181
|
+
# --test wins there, so generated commands stay valid over any config.
|
|
182
|
+
def reject_test_selection_conflicts
|
|
183
|
+
raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
|
|
184
|
+
|
|
185
|
+
reject_test_command_conflicts
|
|
186
|
+
raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
|
|
187
|
+
return unless options[:guess_tests] && options[:tests]&.any?
|
|
188
|
+
|
|
189
|
+
raise UsageError, "--guess-tests cannot be combined with --test"
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# --test-command replaces the whole in-process test machinery: test
|
|
193
|
+
# selection, coverage, parallelism and the sandbox do not apply.
|
|
194
|
+
def reject_test_command_conflicts
|
|
195
|
+
return unless options[:test_command]
|
|
196
|
+
|
|
197
|
+
# options[:tests] is nil or a non-empty list, so truthiness is the check.
|
|
198
|
+
raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
|
|
199
|
+
raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
|
|
151
200
|
end
|
|
152
201
|
|
|
153
202
|
# Usage errors found before the target boots.
|
|
154
203
|
def validate_run_options
|
|
155
|
-
|
|
204
|
+
reject_format_conflicts
|
|
205
|
+
validate_shard_options
|
|
156
206
|
Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
|
|
157
207
|
end
|
|
158
208
|
|
|
159
|
-
# Agent mode
|
|
160
|
-
def
|
|
161
|
-
|
|
209
|
+
# Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
|
|
210
|
+
def reject_format_conflicts
|
|
211
|
+
if agent? && (config["threshold"] || gates.any?)
|
|
212
|
+
raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
|
|
213
|
+
"cannot be combined with --agent"
|
|
214
|
+
end
|
|
215
|
+
return unless @format == "json" && !options[:dry_run]
|
|
162
216
|
|
|
163
|
-
raise UsageError, "
|
|
217
|
+
raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
|
|
164
218
|
end
|
|
165
219
|
|
|
166
220
|
# One line naming the reports written, paths relative to the root when
|
|
167
221
|
# they sit under it (a blank line before it, as the old JSON line had).
|
|
168
222
|
def announce_reports(reports)
|
|
169
|
-
written = reports.select { |path, _label| path }
|
|
170
|
-
return if written.empty?
|
|
223
|
+
return if (written = reports.select { |path, _label| path }).empty?
|
|
171
224
|
|
|
172
225
|
root = File.join(File.realpath(config.root), "")
|
|
173
226
|
names = written.map do |path, label|
|
|
@@ -209,16 +262,38 @@ module Saboteur
|
|
|
209
262
|
Reporter::Agent::Provenance.new(
|
|
210
263
|
ref: config["since"], seed: report.seed, head: tree.head, working_tree: tree.label,
|
|
211
264
|
changed_files: tree.changed_files, mode: agent_mode, json_report: config.path("json_report"),
|
|
212
|
-
subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests ||
|
|
265
|
+
subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
|
|
213
266
|
sources: @sources || [],
|
|
214
267
|
config_path: options[:config], digest: options[:id], cache: config["cache"],
|
|
215
|
-
incremental: config["incremental"], settings: agent_settings
|
|
268
|
+
incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings
|
|
216
269
|
)
|
|
217
270
|
end
|
|
218
271
|
|
|
272
|
+
# The settings keys typed on the command line: a typed empty operator
|
|
273
|
+
# list may clear a config value, so the generated commands keep it.
|
|
274
|
+
def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
|
|
275
|
+
|
|
276
|
+
# A whole-suite run records its scope as --all, so the generated rerun
|
|
277
|
+
# and replay commands do not default back to inference (which can exit
|
|
278
|
+
# 2 in a project with a spec-less source file).
|
|
219
279
|
def agent_settings
|
|
220
|
-
options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
|
|
221
|
-
|
|
280
|
+
settings = options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
|
|
281
|
+
.merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
|
|
282
|
+
# The effective mode: named tests win over a configured whole-suite
|
|
283
|
+
# selection by precedence, so all is recorded only when no tests
|
|
284
|
+
# resolved - otherwise the follow-up commands would carry both --all
|
|
285
|
+
# and --test, which the CLI rejects.
|
|
286
|
+
settings[:all] = true if (config["all"] || config["test_selection"] == "all") && config["tests"].empty?
|
|
287
|
+
# CLI-only switches that change candidate availability or boot, so a
|
|
288
|
+
# printed rerun/replay must keep them (config-file settings like
|
|
289
|
+
# arid/hazard patterns, whole-file, require, rspec_args and rails ride
|
|
290
|
+
# along through --config or the default config path).
|
|
291
|
+
settings[:allow_hazardous] = true if options[:allow_hazardous]
|
|
292
|
+
settings[:allow_focus] = true if options[:allow_focus]
|
|
293
|
+
# An explicit false (--no-max-duration-ok over a config true) is a
|
|
294
|
+
# setting too: dropping it would silently re-enable the config value.
|
|
295
|
+
settings[:max_duration_ok] = options[:max_duration_ok] unless options[:max_duration_ok].nil?
|
|
296
|
+
settings
|
|
222
297
|
end
|
|
223
298
|
|
|
224
299
|
def agent_mode
|
|
@@ -228,40 +303,19 @@ module Saboteur
|
|
|
228
303
|
@sources&.any? ? "files" : "full"
|
|
229
304
|
end
|
|
230
305
|
|
|
231
|
-
# Writes the JSON report and returns its path, or nil when none is configured.
|
|
232
|
-
def write_json(report, cause)
|
|
233
|
-
path = config.path("json_report")
|
|
234
|
-
return unless path
|
|
235
|
-
|
|
236
|
-
Reporter::Stryker.new(path).call(report, exit_cause: cause)
|
|
237
|
-
path
|
|
238
|
-
end
|
|
239
|
-
|
|
240
|
-
def write_html(report)
|
|
241
|
-
path = config.path("html_report")
|
|
242
|
-
return unless path
|
|
243
|
-
|
|
244
|
-
Reporter::Html.new(path).call(report)
|
|
245
|
-
path
|
|
246
|
-
end
|
|
247
|
-
|
|
248
306
|
# Positional arguments name the files (or directories) to mutate. They
|
|
249
307
|
# replace the configured include and exclude patterns. Each must exist,
|
|
250
308
|
# so a typo is an error and not a run over zero subjects.
|
|
251
309
|
def use_explicit_sources
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
options[:tests]&.each { |test| require_existing_test(test) }
|
|
258
|
-
return
|
|
259
|
-
end
|
|
310
|
+
Selection::SourceScope.verify_no_specs!(@argv)
|
|
311
|
+
options[:tests]&.each { |test| require_existing_test(test) }
|
|
312
|
+
# --test may stand alone: the configured include (or the --since
|
|
313
|
+
# diff) names the sources. The files must still exist.
|
|
314
|
+
return if @argv.empty?
|
|
260
315
|
|
|
261
316
|
@sources = @argv.dup
|
|
262
317
|
options[:include] = @argv.map { |source| source_pattern(source) }
|
|
263
318
|
options[:exclude] = []
|
|
264
|
-
options[:tests]&.each { |test| require_existing_test(test) }
|
|
265
319
|
end
|
|
266
320
|
|
|
267
321
|
def source_pattern(source)
|
|
@@ -283,12 +337,10 @@ module Saboteur
|
|
|
283
337
|
raise UsageError, "test file not found: #{test}" unless File.exist?(path)
|
|
284
338
|
end
|
|
285
339
|
|
|
286
|
-
def overrides
|
|
287
|
-
options.except(:config, :id, :suite, :list_operators, :dry_run)
|
|
288
|
-
end
|
|
340
|
+
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run)
|
|
289
341
|
|
|
290
342
|
def preview
|
|
291
|
-
raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
|
|
343
|
+
raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
|
|
292
344
|
raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
|
|
293
345
|
|
|
294
346
|
if options[:list_operators]
|
|
@@ -296,30 +348,14 @@ module Saboteur
|
|
|
296
348
|
return EXIT_OK
|
|
297
349
|
end
|
|
298
350
|
|
|
299
|
-
Preview.new(config, out: @out, err: @err).call
|
|
351
|
+
Preview.new(config, out: @out, err: @err, format: @format).call
|
|
300
352
|
EXIT_OK
|
|
301
353
|
end
|
|
302
354
|
|
|
303
|
-
def define_selection_options(opts)
|
|
304
|
-
opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
|
|
305
|
-
set(:operators, v)
|
|
306
|
-
end
|
|
307
|
-
opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
|
|
308
|
-
set(:exclude_operators, v)
|
|
309
|
-
end
|
|
310
|
-
opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
|
|
311
|
-
opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
|
|
312
|
-
opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
|
|
313
|
-
opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
|
|
314
|
-
opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
|
|
315
|
-
end
|
|
316
|
-
|
|
317
355
|
def define_execution_options(opts)
|
|
318
356
|
opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
|
|
319
357
|
opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
|
|
320
|
-
opts
|
|
321
|
-
set(:seed, v)
|
|
322
|
-
end
|
|
358
|
+
add_run_budget_options(opts)
|
|
323
359
|
opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
|
|
324
360
|
opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
|
|
325
361
|
opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
|
|
@@ -332,8 +368,14 @@ module Saboteur
|
|
|
332
368
|
opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
|
|
333
369
|
set(:allow_focus, true)
|
|
334
370
|
end
|
|
335
|
-
opts.on("--
|
|
336
|
-
|
|
371
|
+
opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
|
|
372
|
+
"mutated files") { set(:all, true) }
|
|
373
|
+
opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
|
|
374
|
+
"For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
|
|
375
|
+
set(:test_command, v)
|
|
376
|
+
end
|
|
377
|
+
opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
|
|
378
|
+
"--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
|
|
337
379
|
set(:guess_tests, v)
|
|
338
380
|
end
|
|
339
381
|
end
|
|
@@ -348,10 +390,10 @@ module Saboteur
|
|
|
348
390
|
opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
|
|
349
391
|
opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
|
|
350
392
|
opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
|
|
351
|
-
opts.on("--format FORMAT", %w[text agent], "text
|
|
352
|
-
"strict exit codes") { |v| @format = v }
|
|
393
|
+
opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
|
|
353
394
|
opts.on("--agent", "Same as --format agent") { @format = "agent" }
|
|
354
|
-
opts.on("--verbose", "Text report: also print score split, timings and kill causes"
|
|
395
|
+
opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
|
|
396
|
+
"a crash prints its backtrace") { @verbose = true }
|
|
355
397
|
opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
|
|
356
398
|
@color = v ? :always : :never
|
|
357
399
|
end
|
|
@@ -367,12 +409,8 @@ module Saboteur
|
|
|
367
409
|
Gates.define_options(opts) { |key, value| set(key, value) }
|
|
368
410
|
end
|
|
369
411
|
|
|
370
|
-
def add_subject(expr)
|
|
371
|
-
(options[:subjects] ||= []) << expr
|
|
372
|
-
end
|
|
412
|
+
def add_subject(expr) = (options[:subjects] ||= []) << expr
|
|
373
413
|
|
|
374
|
-
def set(key, value)
|
|
375
|
-
options[key] = value
|
|
376
|
-
end
|
|
414
|
+
def set(key, value) = (options[key] = value)
|
|
377
415
|
end
|
|
378
416
|
end
|