saboteur 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +199 -0
- data/README.md +427 -20
- data/exe/saboteur +6 -2
- data/lib/saboteur/agent_skill.rb +6 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +106 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +124 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +113 -122
- data/lib/saboteur/config.rb +96 -6
- data/lib/saboteur/doctor.rb +64 -9
- data/lib/saboteur/engine/baseline_guard.rb +33 -11
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +62 -7
- data/lib/saboteur/engine/coverage_store.rb +34 -14
- data/lib/saboteur/engine/diagnostics.rb +150 -4
- data/lib/saboteur/engine/diff_scope.rb +10 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +7 -5
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +92 -20
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/test_guess.rb +89 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +201 -19
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +72 -10
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +101 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/frameworks/database_hosts.rb +30 -0
- data/lib/saboteur/frameworks/rails.rb +8 -10
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +32 -7
- data/lib/saboteur/ignore_list.rb +5 -0
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +21 -1
- data/lib/saboteur/mutation_source/builtin.rb +22 -3
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +35 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +82 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +115 -0
- data/lib/saboteur/reporter/agent_commands.rb +19 -5
- data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
- data/lib/saboteur/reporter/agent_summary.rb +12 -6
- data/lib/saboteur/reporter/findings.rb +4 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_policy.rb +35 -0
- data/lib/saboteur/reporter/history_record.rb +131 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +135 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +18 -3
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +140 -79
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +44 -6
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +8 -0
- data/lib/saboteur.rb +6 -0
- metadata +46 -2
|
@@ -30,9 +30,10 @@ module Saboteur
|
|
|
30
30
|
|
|
31
31
|
# on_start: called in each worker with its number before its first
|
|
32
32
|
# item (per-worker database connections, for example).
|
|
33
|
-
def initialize(jobs:, on_start: nil)
|
|
33
|
+
def initialize(jobs:, on_start: nil, on_result: nil)
|
|
34
34
|
@jobs = jobs
|
|
35
35
|
@on_start = on_start
|
|
36
|
+
@on_result = on_result
|
|
36
37
|
@start_failures = 0
|
|
37
38
|
end
|
|
38
39
|
|
|
@@ -78,13 +79,22 @@ module Saboteur
|
|
|
78
79
|
# next item, retires it when none is left, or replaces it if it died.
|
|
79
80
|
def receive(worker, workers, queue, results, items, &)
|
|
80
81
|
line = worker.results.gets
|
|
81
|
-
|
|
82
|
+
take_and_report(worker, line, results) if line
|
|
82
83
|
return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
|
|
83
84
|
|
|
84
85
|
assign(worker, queue)
|
|
85
86
|
retire(worker, workers) unless worker.current
|
|
86
87
|
end
|
|
87
88
|
|
|
89
|
+
# Records one result line, then reports the arrival before the worker
|
|
90
|
+
# gets more work: a watcher (the canary) can raise here and stop every
|
|
91
|
+
# queued item from starting (issue #102).
|
|
92
|
+
def take_and_report(worker, line, results)
|
|
93
|
+
current = worker.current
|
|
94
|
+
take(worker, line, results)
|
|
95
|
+
@on_result&.call(current, results[current]) if current && results[current]
|
|
96
|
+
end
|
|
97
|
+
|
|
88
98
|
def parse(line)
|
|
89
99
|
JSON.parse(line)
|
|
90
100
|
rescue JSON::ParserError => e
|
|
@@ -112,7 +122,7 @@ module Saboteur
|
|
|
112
122
|
pending = worker.results.read_nonblock(1 << 20, exception: false)
|
|
113
123
|
return unless pending.is_a?(String)
|
|
114
124
|
|
|
115
|
-
pending.each_line { |line|
|
|
125
|
+
pending.each_line { |line| take_and_report(worker, line, results) if line.end_with?("\n") }
|
|
116
126
|
rescue IOError
|
|
117
127
|
nil
|
|
118
128
|
end
|
|
@@ -121,7 +131,13 @@ module Saboteur
|
|
|
121
131
|
drain(worker, results)
|
|
122
132
|
retire(worker, workers)
|
|
123
133
|
reason = worker.last_words || "worker #{worker.number} died#{how(worker.status)}"
|
|
124
|
-
|
|
134
|
+
if worker.current
|
|
135
|
+
# The synthetic error row takes the same arrival path as a normal
|
|
136
|
+
# result, so the canary still trips when the dead worker damaged
|
|
137
|
+
# the tree before dying (issue #102).
|
|
138
|
+
results[worker.current] = { "error" => "#{reason}, running this item" }
|
|
139
|
+
@on_result&.call(worker.current, results[worker.current])
|
|
140
|
+
end
|
|
125
141
|
give_up(reason) if worker.done.zero?
|
|
126
142
|
return if queue.empty?
|
|
127
143
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Execution
|
|
6
|
+
# Registry of gems that wrap a method at definition time. Their wrapper
|
|
7
|
+
# is what Ruby runs, so the shadow guard would call every wrapped method
|
|
8
|
+
# "shadowed" and skip it. A registered wrapper is instead recognized and
|
|
9
|
+
# re-evaluated together with the def (see Scheduler#eval_prefix). An
|
|
10
|
+
# unknown wrapper still hits the generic guard, which is the safe side.
|
|
11
|
+
module Wrapper
|
|
12
|
+
# :sorbet when the runtime method is sorbet-runtime's sig wrapper:
|
|
13
|
+
# its source lives inside the gem and a signature is registered for it.
|
|
14
|
+
def self.around(method, file)
|
|
15
|
+
return nil unless file&.include?("sorbet-runtime")
|
|
16
|
+
return nil unless defined?(::T::Utils) && ::T::Utils.respond_to?(:signature_for_method)
|
|
17
|
+
return nil unless ::T::Utils.signature_for_method(method)
|
|
18
|
+
|
|
19
|
+
:sorbet
|
|
20
|
+
rescue StandardError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The def a wrapper dispatches to: sorbet's runtime method is a
|
|
25
|
+
# forwarding wrapper, while the registered signature holds the real
|
|
26
|
+
# def. Callers comparing definitions (StaleCaptures) must compare
|
|
27
|
+
# against this, or the freshly re-registered def reads as stale.
|
|
28
|
+
# Real sorbet returns a Signature object (its #method is the def);
|
|
29
|
+
# minimal registries return the method itself. Nil when unknown.
|
|
30
|
+
def self.original_def(method)
|
|
31
|
+
return nil unless around(method, method.source_location&.first) == :sorbet
|
|
32
|
+
|
|
33
|
+
registered = ::T::Utils.signature_for_method(method)
|
|
34
|
+
candidate = if registered.is_a?(Method) || registered.is_a?(UnboundMethod)
|
|
35
|
+
registered
|
|
36
|
+
elsif registered.respond_to?(:method)
|
|
37
|
+
registered.method
|
|
38
|
+
end
|
|
39
|
+
candidate.is_a?(Method) || candidate.is_a?(UnboundMethod) ? candidate : nil
|
|
40
|
+
rescue StandardError
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Frameworks
|
|
6
|
+
# Which database hosts count as local for the Rails integration's refusal
|
|
7
|
+
# to mutate a remote test database (issue #258).
|
|
8
|
+
module DatabaseHosts
|
|
9
|
+
# 0.0.0.0: a listen address, but in a URL it connects to this machine (issue #258).
|
|
10
|
+
LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]", "0.0.0.0"].freeze
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def local?(host)
|
|
15
|
+
host = host.to_s
|
|
16
|
+
host.empty? || host.start_with?("/") || LOCAL_HOSTS.include?(host.downcase)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# "name (host host)" per config whose host is not local and not in
|
|
20
|
+
# extra_allowed. Shared by the boot refusal and doctor.
|
|
21
|
+
def remote(configs, extra_allowed = [])
|
|
22
|
+
allowed = extra_allowed.map(&:downcase)
|
|
23
|
+
configs.filter_map do |db|
|
|
24
|
+
host = db.configuration_hash[:host]
|
|
25
|
+
"#{db.name} (host #{host})" unless local?(host) || allowed.include?(host.downcase)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -7,6 +7,8 @@ require "pathname"
|
|
|
7
7
|
require "English"
|
|
8
8
|
require "tmpdir"
|
|
9
9
|
|
|
10
|
+
require_relative "database_hosts"
|
|
11
|
+
|
|
10
12
|
module Saboteur
|
|
11
13
|
module Frameworks
|
|
12
14
|
# Opt-in Rails support (`c.rails = true` in config/saboteur.rb).
|
|
@@ -16,8 +18,6 @@ module Saboteur
|
|
|
16
18
|
# forking, and drops database connections in the parent so each child opens
|
|
17
19
|
# its own. Prints host names only, never credentials.
|
|
18
20
|
class Rails
|
|
19
|
-
LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
|
|
20
|
-
|
|
21
21
|
# How many blocking session names the template-copy error lists.
|
|
22
22
|
MAX_BLOCKER_NAMES = 5
|
|
23
23
|
|
|
@@ -170,9 +170,9 @@ module Saboteur
|
|
|
170
170
|
db.adapter.to_s.match?(/sqlite/) && db.database.to_s == ":memory:"
|
|
171
171
|
end
|
|
172
172
|
|
|
173
|
-
def self.local_host?(host)
|
|
174
|
-
|
|
175
|
-
|
|
173
|
+
def self.local_host?(host) = DatabaseHosts.local?(host)
|
|
174
|
+
|
|
175
|
+
def self.remote_databases(configs, extra_allowed = []) = DatabaseHosts.remote(configs, extra_allowed)
|
|
176
176
|
|
|
177
177
|
private
|
|
178
178
|
|
|
@@ -197,13 +197,11 @@ module Saboteur
|
|
|
197
197
|
def check_databases
|
|
198
198
|
return unless defined?(::ActiveRecord::Base)
|
|
199
199
|
|
|
200
|
-
remote =
|
|
201
|
-
host = db.configuration_hash[:host]
|
|
202
|
-
"#{db.name} (host #{host})" unless self.class.local_host?(host)
|
|
203
|
-
end
|
|
200
|
+
remote = self.class.remote_databases(test_configs, @config["allow_db_hosts"])
|
|
204
201
|
return if remote.empty?
|
|
205
202
|
|
|
206
|
-
raise Error, "rails integration refuses to run against non-local databases: #{remote.join(', ')}"
|
|
203
|
+
raise Error, "rails integration refuses to run against non-local databases: #{remote.join(', ')}; " \
|
|
204
|
+
"if they are disposable test databases, allow them with c.allow_db_hosts in #{Config::FILE}"
|
|
207
205
|
end
|
|
208
206
|
|
|
209
207
|
def test_configs
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# CI gates on how many mutants ended as a timeout or an error. A limit is a
|
|
6
|
+
# count ("3") or a percentage of the executed mutants ("5%"); the gate fails
|
|
7
|
+
# only when the count is above it. Unset means no gate.
|
|
8
|
+
class Gates
|
|
9
|
+
Violation = Data.define(:cause, :message)
|
|
10
|
+
|
|
11
|
+
# config key => [mutant status, exit cause recorded in the JSON report, flag]
|
|
12
|
+
SETTINGS = {
|
|
13
|
+
"max_timeouts" => [:timeout, "maxTimeouts", "--max-timeouts"],
|
|
14
|
+
"max_errors" => [:error, "maxErrors", "--max-errors"]
|
|
15
|
+
}.freeze
|
|
16
|
+
LIMIT = /\A(\d+)(%?)\z/
|
|
17
|
+
|
|
18
|
+
# The normalized limit: nil, an Integer count, or a String ending in "%".
|
|
19
|
+
# A malformed value is a ConfigError that names the setting.
|
|
20
|
+
def self.limit(value, key)
|
|
21
|
+
return nil if value.nil?
|
|
22
|
+
|
|
23
|
+
match = LIMIT.match(value.to_s.strip)
|
|
24
|
+
raise ConfigError, "#{key} must be a count (3) or a percentage (5%), got #{value.inspect}" unless match
|
|
25
|
+
|
|
26
|
+
digits, suffix = match.captures
|
|
27
|
+
number = digits.to_i
|
|
28
|
+
raise ConfigError, "#{key} percentage must be at most 100" if suffix == "%" && number > 100
|
|
29
|
+
|
|
30
|
+
suffix.empty? ? number : "#{number}%"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The --max-timeouts and --max-errors flags; each value goes to the block with its config key.
|
|
34
|
+
def self.define_options(parser)
|
|
35
|
+
SETTINGS.each do |key, (status, _cause, flag)|
|
|
36
|
+
parser.on("#{flag} N", "Exit 1 when more than N mutants end as #{status} (N or N%)") do |value|
|
|
37
|
+
yield key.to_sym, value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def initialize(config)
|
|
43
|
+
@limits = SETTINGS.keys.to_h { |key| [key, self.class.limit(config[key], key)] }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
attr_reader :cause
|
|
47
|
+
|
|
48
|
+
def any?
|
|
49
|
+
@limits.values.any?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# --max-errors judges errored mutants, so a threshold run tolerates them.
|
|
53
|
+
def judges_errors?
|
|
54
|
+
!@limits.fetch("max_errors").nil?
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Prints each failed gate to err and remembers the first one's cause for the JSON report.
|
|
58
|
+
def failed?(report, err)
|
|
59
|
+
found = violations(report)
|
|
60
|
+
found.each { |violation| err.puts("saboteur: #{violation.message}") }
|
|
61
|
+
@cause = found.first&.cause
|
|
62
|
+
!@cause.nil?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def violations(report)
|
|
66
|
+
SETTINGS.filter_map do |key, (status, cause, flag)|
|
|
67
|
+
limit = @limits.fetch(key)
|
|
68
|
+
count = report.counts.fetch(status)
|
|
69
|
+
next if limit.nil? || !over?(count, limit, report.executed.size)
|
|
70
|
+
|
|
71
|
+
Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def over?(count, limit, executed)
|
|
78
|
+
return count > limit if limit.is_a?(Integer)
|
|
79
|
+
|
|
80
|
+
count * 100 > limit.to_i * executed
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -21,7 +21,7 @@ module Saboteur
|
|
|
21
21
|
"errored. With --threshold: the score is at or above it, even if mutants survived.",
|
|
22
22
|
1 => "Without --threshold: at least one mutant survived and is not on the ignore list. With --threshold " \
|
|
23
23
|
"(survivors alone no longer fail the run): the score is below it, or no mutant was tested so " \
|
|
24
|
-
"there is no score.",
|
|
24
|
+
"there is no score. Also more timeout or errored mutants than --max-timeouts or --max-errors allow.",
|
|
25
25
|
2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
|
|
26
26
|
"mutant is only found after the target has booted. A named source or test file that does not exist.",
|
|
27
27
|
3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
|
|
@@ -42,7 +42,9 @@ module Saboteur
|
|
|
42
42
|
"tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
|
|
43
43
|
"directories after run replace include and exclude; --test limits the specs that run; " \
|
|
44
44
|
"--threshold PERCENT replaces the survivor check: exit 0 when the score is at or above " \
|
|
45
|
-
"PERCENT, exit 1 when it is below."
|
|
45
|
+
"PERCENT, exit 1 when it is below. All operators run unless you select a smaller set; " \
|
|
46
|
+
"exclusions always remove operators from that set. Inline comment syntax: " \
|
|
47
|
+
'# saboteur:disable-line [operator ...] reason="...". A reason is required.',
|
|
46
48
|
takes_options: true,
|
|
47
49
|
exit_codes: EXIT_CODE_MEANINGS,
|
|
48
50
|
examples: [
|
|
@@ -58,14 +60,37 @@ module Saboteur
|
|
|
58
60
|
),
|
|
59
61
|
Command.new(
|
|
60
62
|
name: "doctor",
|
|
61
|
-
summary: "Check the environment and run the baseline twice, without mutating anything",
|
|
62
|
-
description: "Boots the target like a run
|
|
63
|
-
"suite twice in the same order with one seed and reports whether both
|
|
64
|
-
"It never mutates anything.",
|
|
63
|
+
summary: "Check the environment, and with --suite run the baseline twice, without mutating anything",
|
|
64
|
+
description: "Boots the target like a run and checks runtime, coverage and boot files. With --suite " \
|
|
65
|
+
"it also runs your suite twice in the same order with one seed and reports whether both " \
|
|
66
|
+
"runs passed; --test PATH limits those runs to some specs. It never mutates anything.",
|
|
65
67
|
takes_options: true,
|
|
66
68
|
exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
|
|
67
69
|
examples: [["Check the setup before the first run", "saboteur doctor"],
|
|
68
|
-
["
|
|
70
|
+
["Also run the unchanged suite twice", "saboteur doctor --suite --seed 7"],
|
|
71
|
+
["Run only some specs twice", "saboteur doctor --test spec/models"]]
|
|
72
|
+
),
|
|
73
|
+
Command.new(
|
|
74
|
+
name: "baseline",
|
|
75
|
+
summary: "Check, propose or accept a fresh opt-in survivor baseline",
|
|
76
|
+
description: "Check runs every mutant fresh and reports new, changed or unmatched accepted survivors. " \
|
|
77
|
+
"Propose writes a proposal with private local execution evidence. Edit only survivor reasons; " \
|
|
78
|
+
"accept verifies current inputs and publishes it. No scores or execution are suppressed. " \
|
|
79
|
+
"With --epsilon POINTS, check also fails when the score drops more than POINTS percentage " \
|
|
80
|
+
"points below the accepted baseline's score. " \
|
|
81
|
+
"Only whole or subject scope; no diff, replay, incremental reuse or score threshold.",
|
|
82
|
+
takes_options: true,
|
|
83
|
+
exit_codes: { 0 => "Clean check, proposal written or accepted.",
|
|
84
|
+
1 => "New, changed or unmatched survivors, or a score drop beyond --epsilon.",
|
|
85
|
+
2 => "Invalid configuration, incompatible baseline or rejected acceptance.",
|
|
86
|
+
3 => "Incomplete execution or tool failure." },
|
|
87
|
+
examples: [["Prepare a reasoned proposal",
|
|
88
|
+
"saboteur baseline propose --baseline baseline.json --proposal proposal.json"],
|
|
89
|
+
["Publish reviewed reasons",
|
|
90
|
+
"saboteur baseline accept --baseline baseline.json --proposal proposal.json"],
|
|
91
|
+
["Check fresh outcomes", "saboteur baseline check --baseline baseline.json"],
|
|
92
|
+
["Also fail on a score drop of more than 2 points",
|
|
93
|
+
"saboteur baseline check --baseline baseline.json --epsilon 2"]]
|
|
69
94
|
),
|
|
70
95
|
Command.new(
|
|
71
96
|
name: "version",
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -96,6 +96,11 @@ module Saboteur
|
|
|
96
96
|
@index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
+
# True when this result is a survivor the list accepts.
|
|
100
|
+
def accepts?(result)
|
|
101
|
+
result.survived? && @index.key?([result.subject.id, result.digest])
|
|
102
|
+
end
|
|
103
|
+
|
|
99
104
|
# Marks matching survivors as ignored. Returns warnings for stale entries:
|
|
100
105
|
# entries whose subject was examined but whose mutation no longer exists.
|
|
101
106
|
def apply(results)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require "shellwords"
|
|
6
|
+
require_relative "mutation_source/operator_registry"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
# Only real Ruby comments can suppress edits, never text inside a string.
|
|
10
|
+
class InlineSuppression
|
|
11
|
+
Directive = Data.define(:line, :operators, :reason)
|
|
12
|
+
|
|
13
|
+
def initialize(file, source)
|
|
14
|
+
@file = file
|
|
15
|
+
@directives = Prism.parse(source).comments.filter_map { |comment| parse(comment) }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Preserve ordinary comment/formatting skips, but treat directives as
|
|
19
|
+
# selection inputs. Relative lines distinguish which code they suppress.
|
|
20
|
+
def signature(subject)
|
|
21
|
+
@directives.select { |entry| subject.contains_line?(entry.line) }.map do |entry|
|
|
22
|
+
[entry.line - subject.start_line, entry.operators.sort, entry.reason]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def reason(subject, mutation)
|
|
27
|
+
mutation.edits.each do |edit|
|
|
28
|
+
line = subject.line_of(edit.start_byte)
|
|
29
|
+
directive = @directives.find do |entry|
|
|
30
|
+
entry.line == line && (entry.operators.empty? || entry.operators.include?(mutation.operator))
|
|
31
|
+
end
|
|
32
|
+
return directive if directive
|
|
33
|
+
end
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def parse(comment)
|
|
40
|
+
text = comment.location.slice.sub(/\A#\s*/, "")
|
|
41
|
+
return nil unless text.start_with?("saboteur:disable-line")
|
|
42
|
+
|
|
43
|
+
line = comment.location.start_line
|
|
44
|
+
tokens = Shellwords.split(text)
|
|
45
|
+
fail_at(line, "expected saboteur:disable-line") unless tokens.shift == "saboteur:disable-line"
|
|
46
|
+
reasons, operators = tokens.partition { |token| token.start_with?("reason=") }
|
|
47
|
+
reason, = reasons
|
|
48
|
+
fail_at(line, "one nonblank reason=\"...\" is required") unless
|
|
49
|
+
reasons.size == 1 && !reason.delete_prefix("reason=").strip.empty?
|
|
50
|
+
validate_operators(operators, line)
|
|
51
|
+
Directive.new(line: line, operators: operators, reason: reason.delete_prefix("reason="))
|
|
52
|
+
rescue ArgumentError => e
|
|
53
|
+
fail_at(line, "invalid suppression: #{e.message}")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def validate_operators(operators, line)
|
|
57
|
+
unknown = operators - MutationSource::OperatorRegistry::DESCRIPTIONS.keys
|
|
58
|
+
fail_at(line, "unknown operators: #{unknown.join(', ')}") unless unknown.empty?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def fail_at(line, message)
|
|
62
|
+
raise ConfigError, "#{@file}:#{line}: #{message}"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Integration
|
|
6
|
+
# Refuses a suite that a focus filter has narrowed without the user asking
|
|
7
|
+
# for it. With `filter_run_when_matching :focus`, one stray `fit` makes
|
|
8
|
+
# RSpec skip every other example. The baseline stays green and the score
|
|
9
|
+
# then describes only the focused examples, with no sign that the rest
|
|
10
|
+
# never ran.
|
|
11
|
+
#
|
|
12
|
+
# The test is a difference of two selections, both made by RSpec's own
|
|
13
|
+
# FilterManager#prune: the examples that run now, and the examples that
|
|
14
|
+
# would run if the stray focus rule were absent. Everything else RSpec
|
|
15
|
+
# decides stays the same in both, so it never counts as hidden: example
|
|
16
|
+
# selectors (`spec/a_spec.rb:12`) and their file-scoped priority,
|
|
17
|
+
# `--tag` filters named in rspec_args, exclusions, and the :if / :unless
|
|
18
|
+
# pruning of conditional examples.
|
|
19
|
+
class FocusGuard
|
|
20
|
+
LOCATION_KEYS = %i[locations ids].freeze
|
|
21
|
+
|
|
22
|
+
def initialize(world:, explicit_filter_keys:)
|
|
23
|
+
@world = world
|
|
24
|
+
@explicit_filter_keys = explicit_filter_keys
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Raises ConfigError naming the focused examples when the focus filter
|
|
28
|
+
# hides examples that would otherwise run.
|
|
29
|
+
def check!
|
|
30
|
+
return if stray_rules.empty?
|
|
31
|
+
return if hidden_examples.empty?
|
|
32
|
+
|
|
33
|
+
raise ConfigError, message
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
attr_reader :world
|
|
39
|
+
|
|
40
|
+
def filter_manager
|
|
41
|
+
world.filter_manager
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Inclusion rules the suite added itself, such as the :focus rule that
|
|
45
|
+
# `filter_run_when_matching :focus` installs once a fit exists.
|
|
46
|
+
def stray_rules
|
|
47
|
+
@stray_rules ||= begin
|
|
48
|
+
rules = filter_manager.inclusions.rules
|
|
49
|
+
rules.slice(*(rules.keys - LOCATION_KEYS - @explicit_filter_keys))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def running_examples
|
|
54
|
+
@running_examples ||= filter_manager.prune(world.all_examples)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The selection RSpec would make without the stray rules.
|
|
58
|
+
def examples_without_stray_focus
|
|
59
|
+
manager = ::RSpec::Core::FilterManager.new
|
|
60
|
+
manager.exclusions.rules.replace(filter_manager.exclusions.rules)
|
|
61
|
+
manager.inclusions.rules.replace(filter_manager.inclusions.rules.except(*stray_rules.keys))
|
|
62
|
+
manager.prune(world.all_examples)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def hidden_examples
|
|
66
|
+
@hidden_examples ||= examples_without_stray_focus - running_examples
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def focused_examples
|
|
70
|
+
focus = ::RSpec::Core::FilterRules.new(stray_rules)
|
|
71
|
+
running_examples.select { |example| focus.include_example?(example) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def message
|
|
75
|
+
total = hidden_examples.size + running_examples.size
|
|
76
|
+
locations = focused_examples.map(&:location).uniq.first(5).join(", ")
|
|
77
|
+
"a focus filter (#{::RSpec::Core::FilterRules.new(stray_rules).description}) narrowed the suite: " \
|
|
78
|
+
"#{hidden_examples.size} of #{total} examples would not run, so the score would cover only the " \
|
|
79
|
+
"ones that do (#{locations}). Remove the focus, or pass --allow-focus to run the narrowed suite anyway."
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -14,6 +14,8 @@ module Saboteur
|
|
|
14
14
|
@runner.setup($stderr, $stdout)
|
|
15
15
|
configuration.order = "rand:#{config.fetch('seed', 1)}"
|
|
16
16
|
raise Error, "rspec quit while loading spec files" if world.wants_to_quit
|
|
17
|
+
|
|
18
|
+
refuse_unintended_focus
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
# RSpec only adds its default "spec" path when $0 is `rspec`.
|
|
@@ -48,7 +50,7 @@ module Saboteur
|
|
|
48
50
|
wanted = Set.new(ids)
|
|
49
51
|
narrow_to(wanted)
|
|
50
52
|
order_by(ids)
|
|
51
|
-
configuration.fail_fast = 1
|
|
53
|
+
configuration.fail_fast = config["matrix"] ? false : 1
|
|
52
54
|
listener = Listener.new
|
|
53
55
|
passed = execute(listener)
|
|
54
56
|
RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
|
|
@@ -74,6 +76,24 @@ module Saboteur
|
|
|
74
76
|
|
|
75
77
|
private
|
|
76
78
|
|
|
79
|
+
def refuse_unintended_focus
|
|
80
|
+
return if config["allow_focus"]
|
|
81
|
+
|
|
82
|
+
FocusGuard.new(world:, explicit_filter_keys: explicit_inclusion_keys).check!
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Inclusion filters the user named in rspec_args or .rspec, such as
|
|
86
|
+
# `--tag slow`. Those narrow the run on purpose.
|
|
87
|
+
def explicit_inclusion_keys
|
|
88
|
+
options = configuration_options
|
|
89
|
+
filter_manager = ::RSpec::Core::FilterManager.new
|
|
90
|
+
options.configure_filter_manager(filter_manager)
|
|
91
|
+
keys = filter_manager.inclusions.rules.keys
|
|
92
|
+
# `--example TEXT` becomes a :full_description rule when the options
|
|
93
|
+
# reach the configuration, not in the filter manager.
|
|
94
|
+
options.options[:full_description] ? keys + [:full_description] : keys
|
|
95
|
+
end
|
|
96
|
+
|
|
77
97
|
def world
|
|
78
98
|
::RSpec.world
|
|
79
99
|
end
|
|
@@ -6,7 +6,13 @@ require_relative "base"
|
|
|
6
6
|
require_relative "filter_replacement"
|
|
7
7
|
require_relative "literal_replacement"
|
|
8
8
|
require_relative "argument_removal"
|
|
9
|
+
require_relative "chain_removal"
|
|
9
10
|
require_relative "pair_removal"
|
|
11
|
+
require_relative "endpoint_replacement"
|
|
12
|
+
require_relative "operator_registry"
|
|
13
|
+
require_relative "range_replacement"
|
|
14
|
+
require_relative "operator_assignment"
|
|
15
|
+
require_relative "condition_negation"
|
|
10
16
|
|
|
11
17
|
module Saboteur
|
|
12
18
|
module MutationSource
|
|
@@ -15,7 +21,7 @@ module Saboteur
|
|
|
15
21
|
class Builtin < Base
|
|
16
22
|
DEFAULT_ARID = [
|
|
17
23
|
'\A(puts|pp|p|print|warn)\b',
|
|
18
|
-
'\A(
|
|
24
|
+
'\A(@|self\.|::)?(Rails\.)?logger\b',
|
|
19
25
|
'\A(::)?(StatsD|Statsd|Datadog|NewRelic)\b',
|
|
20
26
|
'\.instrument\b'
|
|
21
27
|
].freeze
|
|
@@ -47,13 +53,19 @@ module Saboteur
|
|
|
47
53
|
CATCH_ALL = %w[StandardError ::StandardError Exception ::Exception].freeze
|
|
48
54
|
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
49
55
|
|
|
50
|
-
def initialize(arid: DEFAULT_ARID)
|
|
56
|
+
def initialize(arid: DEFAULT_ARID, operators: [], exclude_operators: [])
|
|
51
57
|
super()
|
|
58
|
+
@operators = OperatorRegistry.select(operators, exclude_operators)
|
|
52
59
|
@arid = arid.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) }
|
|
53
60
|
@filter_replacement = FilterReplacement.new(arid: arid)
|
|
54
61
|
@literal_replacement = LiteralReplacement.new(arid: arid)
|
|
55
62
|
@argument_removal = ArgumentRemoval.new(arid: arid)
|
|
56
63
|
@pair_removal = PairRemoval.new(arid: arid)
|
|
64
|
+
@chain_removal = ChainRemoval.new(arid: arid)
|
|
65
|
+
@endpoint_replacement = EndpointReplacement.new(arid: arid)
|
|
66
|
+
@range_replacement = RangeReplacement.new(arid: arid)
|
|
67
|
+
@operator_assignment = OperatorAssignment.new(arid: arid)
|
|
68
|
+
@condition_negation = ConditionNegation.new(arid: arid)
|
|
57
69
|
end
|
|
58
70
|
|
|
59
71
|
def name
|
|
@@ -70,6 +82,12 @@ module Saboteur
|
|
|
70
82
|
@found.concat(@literal_replacement.mutations_for(subject))
|
|
71
83
|
@found.concat(@argument_removal.mutations_for(subject))
|
|
72
84
|
@found.concat(@pair_removal.mutations_for(subject))
|
|
85
|
+
@found.concat(@chain_removal.mutations_for(subject))
|
|
86
|
+
@found.concat(@endpoint_replacement.mutations_for(subject))
|
|
87
|
+
@found.concat(@range_replacement.mutations_for(subject))
|
|
88
|
+
@found.concat(@operator_assignment.mutations_for(subject))
|
|
89
|
+
@found.concat(@condition_negation.mutations_for(subject))
|
|
90
|
+
@found.select { |mutation| @operators.include?(mutation.operator) }
|
|
73
91
|
ensure
|
|
74
92
|
@subject = nil
|
|
75
93
|
end
|
|
@@ -134,7 +152,8 @@ module Saboteur
|
|
|
134
152
|
statements.each do |statement|
|
|
135
153
|
next if arid?(statement)
|
|
136
154
|
|
|
137
|
-
|
|
155
|
+
removable_here = statements.drop(1).any? && removable?(statement)
|
|
156
|
+
add("statement_removal", statement, "nil") if removable_here
|
|
138
157
|
visit(statement)
|
|
139
158
|
end
|
|
140
159
|
end
|