saboteur 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +167 -0
- data/README.md +393 -20
- data/exe/saboteur +6 -2
- data/lib/saboteur/agent_skill.rb +6 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +105 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +114 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +101 -121
- data/lib/saboteur/config.rb +76 -6
- data/lib/saboteur/doctor.rb +40 -8
- data/lib/saboteur/engine/baseline_guard.rb +33 -11
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +59 -7
- data/lib/saboteur/engine/coverage_store.rb +34 -14
- data/lib/saboteur/engine/diagnostics.rb +150 -4
- data/lib/saboteur/engine/diff_scope.rb +10 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +6 -4
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +92 -20
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/test_guess.rb +52 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +192 -18
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +72 -10
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +101 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +31 -7
- data/lib/saboteur/ignore_list.rb +5 -0
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +21 -1
- data/lib/saboteur/mutation_source/builtin.rb +22 -3
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +106 -0
- data/lib/saboteur/reporter/agent_commands.rb +7 -1
- data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
- data/lib/saboteur/reporter/agent_summary.rb +11 -5
- data/lib/saboteur/reporter/findings.rb +4 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_record.rb +132 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +123 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +15 -2
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +131 -79
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +41 -6
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +8 -0
- data/lib/saboteur.rb +6 -0
- metadata +43 -2
|
@@ -30,9 +30,10 @@ module Saboteur
|
|
|
30
30
|
|
|
31
31
|
# on_start: called in each worker with its number before its first
|
|
32
32
|
# item (per-worker database connections, for example).
|
|
33
|
-
def initialize(jobs:, on_start: nil)
|
|
33
|
+
def initialize(jobs:, on_start: nil, on_result: nil)
|
|
34
34
|
@jobs = jobs
|
|
35
35
|
@on_start = on_start
|
|
36
|
+
@on_result = on_result
|
|
36
37
|
@start_failures = 0
|
|
37
38
|
end
|
|
38
39
|
|
|
@@ -78,13 +79,22 @@ module Saboteur
|
|
|
78
79
|
# next item, retires it when none is left, or replaces it if it died.
|
|
79
80
|
def receive(worker, workers, queue, results, items, &)
|
|
80
81
|
line = worker.results.gets
|
|
81
|
-
|
|
82
|
+
take_and_report(worker, line, results) if line
|
|
82
83
|
return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
|
|
83
84
|
|
|
84
85
|
assign(worker, queue)
|
|
85
86
|
retire(worker, workers) unless worker.current
|
|
86
87
|
end
|
|
87
88
|
|
|
89
|
+
# Records one result line, then reports the arrival before the worker
|
|
90
|
+
# gets more work: a watcher (the canary) can raise here and stop every
|
|
91
|
+
# queued item from starting (issue #102).
|
|
92
|
+
def take_and_report(worker, line, results)
|
|
93
|
+
current = worker.current
|
|
94
|
+
take(worker, line, results)
|
|
95
|
+
@on_result&.call(current, results[current]) if current && results[current]
|
|
96
|
+
end
|
|
97
|
+
|
|
88
98
|
def parse(line)
|
|
89
99
|
JSON.parse(line)
|
|
90
100
|
rescue JSON::ParserError => e
|
|
@@ -112,7 +122,7 @@ module Saboteur
|
|
|
112
122
|
pending = worker.results.read_nonblock(1 << 20, exception: false)
|
|
113
123
|
return unless pending.is_a?(String)
|
|
114
124
|
|
|
115
|
-
pending.each_line { |line|
|
|
125
|
+
pending.each_line { |line| take_and_report(worker, line, results) if line.end_with?("\n") }
|
|
116
126
|
rescue IOError
|
|
117
127
|
nil
|
|
118
128
|
end
|
|
@@ -121,7 +131,13 @@ module Saboteur
|
|
|
121
131
|
drain(worker, results)
|
|
122
132
|
retire(worker, workers)
|
|
123
133
|
reason = worker.last_words || "worker #{worker.number} died#{how(worker.status)}"
|
|
124
|
-
|
|
134
|
+
if worker.current
|
|
135
|
+
# The synthetic error row takes the same arrival path as a normal
|
|
136
|
+
# result, so the canary still trips when the dead worker damaged
|
|
137
|
+
# the tree before dying (issue #102).
|
|
138
|
+
results[worker.current] = { "error" => "#{reason}, running this item" }
|
|
139
|
+
@on_result&.call(worker.current, results[worker.current])
|
|
140
|
+
end
|
|
125
141
|
give_up(reason) if worker.done.zero?
|
|
126
142
|
return if queue.empty?
|
|
127
143
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Execution
|
|
6
|
+
# Registry of gems that wrap a method at definition time. Their wrapper
|
|
7
|
+
# is what Ruby runs, so the shadow guard would call every wrapped method
|
|
8
|
+
# "shadowed" and skip it. A registered wrapper is instead recognized and
|
|
9
|
+
# re-evaluated together with the def (see Scheduler#eval_prefix). An
|
|
10
|
+
# unknown wrapper still hits the generic guard, which is the safe side.
|
|
11
|
+
module Wrapper
|
|
12
|
+
# :sorbet when the runtime method is sorbet-runtime's sig wrapper:
|
|
13
|
+
# its source lives inside the gem and a signature is registered for it.
|
|
14
|
+
def self.around(method, file)
|
|
15
|
+
return nil unless file&.include?("sorbet-runtime")
|
|
16
|
+
return nil unless defined?(::T::Utils) && ::T::Utils.respond_to?(:signature_for_method)
|
|
17
|
+
return nil unless ::T::Utils.signature_for_method(method)
|
|
18
|
+
|
|
19
|
+
:sorbet
|
|
20
|
+
rescue StandardError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The def a wrapper dispatches to: sorbet's runtime method is a
|
|
25
|
+
# forwarding wrapper, while the registered signature holds the real
|
|
26
|
+
# def. Callers comparing definitions (StaleCaptures) must compare
|
|
27
|
+
# against this, or the freshly re-registered def reads as stale.
|
|
28
|
+
# Real sorbet returns a Signature object (its #method is the def);
|
|
29
|
+
# minimal registries return the method itself. Nil when unknown.
|
|
30
|
+
def self.original_def(method)
|
|
31
|
+
return nil unless around(method, method.source_location&.first) == :sorbet
|
|
32
|
+
|
|
33
|
+
registered = ::T::Utils.signature_for_method(method)
|
|
34
|
+
candidate = if registered.is_a?(Method) || registered.is_a?(UnboundMethod)
|
|
35
|
+
registered
|
|
36
|
+
elsif registered.respond_to?(:method)
|
|
37
|
+
registered.method
|
|
38
|
+
end
|
|
39
|
+
candidate.is_a?(Method) || candidate.is_a?(UnboundMethod) ? candidate : nil
|
|
40
|
+
rescue StandardError
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# CI gates on how many mutants ended as a timeout or an error. A limit is a
|
|
6
|
+
# count ("3") or a percentage of the executed mutants ("5%"); the gate fails
|
|
7
|
+
# only when the count is above it. Unset means no gate.
|
|
8
|
+
class Gates
|
|
9
|
+
Violation = Data.define(:cause, :message)
|
|
10
|
+
|
|
11
|
+
# config key => [mutant status, exit cause recorded in the JSON report, flag]
|
|
12
|
+
SETTINGS = {
|
|
13
|
+
"max_timeouts" => [:timeout, "maxTimeouts", "--max-timeouts"],
|
|
14
|
+
"max_errors" => [:error, "maxErrors", "--max-errors"]
|
|
15
|
+
}.freeze
|
|
16
|
+
LIMIT = /\A(\d+)(%?)\z/
|
|
17
|
+
|
|
18
|
+
# The normalized limit: nil, an Integer count, or a String ending in "%".
|
|
19
|
+
# A malformed value is a ConfigError that names the setting.
|
|
20
|
+
def self.limit(value, key)
|
|
21
|
+
return nil if value.nil?
|
|
22
|
+
|
|
23
|
+
match = LIMIT.match(value.to_s.strip)
|
|
24
|
+
raise ConfigError, "#{key} must be a count (3) or a percentage (5%), got #{value.inspect}" unless match
|
|
25
|
+
|
|
26
|
+
digits, suffix = match.captures
|
|
27
|
+
number = digits.to_i
|
|
28
|
+
raise ConfigError, "#{key} percentage must be at most 100" if suffix == "%" && number > 100
|
|
29
|
+
|
|
30
|
+
suffix.empty? ? number : "#{number}%"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The --max-timeouts and --max-errors flags; each value goes to the block with its config key.
|
|
34
|
+
def self.define_options(parser)
|
|
35
|
+
SETTINGS.each do |key, (status, _cause, flag)|
|
|
36
|
+
parser.on("#{flag} N", "Exit 1 when more than N mutants end as #{status} (N or N%)") do |value|
|
|
37
|
+
yield key.to_sym, value
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def initialize(config)
|
|
43
|
+
@limits = SETTINGS.keys.to_h { |key| [key, self.class.limit(config[key], key)] }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
attr_reader :cause
|
|
47
|
+
|
|
48
|
+
def any?
|
|
49
|
+
@limits.values.any?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# --max-errors judges errored mutants, so a threshold run tolerates them.
|
|
53
|
+
def judges_errors?
|
|
54
|
+
!@limits.fetch("max_errors").nil?
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Prints each failed gate to err and remembers the first one's cause for the JSON report.
|
|
58
|
+
def failed?(report, err)
|
|
59
|
+
found = violations(report)
|
|
60
|
+
found.each { |violation| err.puts("saboteur: #{violation.message}") }
|
|
61
|
+
@cause = found.first&.cause
|
|
62
|
+
!@cause.nil?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def violations(report)
|
|
66
|
+
SETTINGS.filter_map do |key, (status, cause, flag)|
|
|
67
|
+
limit = @limits.fetch(key)
|
|
68
|
+
count = report.counts.fetch(status)
|
|
69
|
+
next if limit.nil? || !over?(count, limit, report.executed.size)
|
|
70
|
+
|
|
71
|
+
Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def over?(count, limit, executed)
|
|
78
|
+
return count > limit if limit.is_a?(Integer)
|
|
79
|
+
|
|
80
|
+
count * 100 > limit.to_i * executed
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -21,7 +21,7 @@ module Saboteur
|
|
|
21
21
|
"errored. With --threshold: the score is at or above it, even if mutants survived.",
|
|
22
22
|
1 => "Without --threshold: at least one mutant survived and is not on the ignore list. With --threshold " \
|
|
23
23
|
"(survivors alone no longer fail the run): the score is below it, or no mutant was tested so " \
|
|
24
|
-
"there is no score.",
|
|
24
|
+
"there is no score. Also more timeout or errored mutants than --max-timeouts or --max-errors allow.",
|
|
25
25
|
2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
|
|
26
26
|
"mutant is only found after the target has booted. A named source or test file that does not exist.",
|
|
27
27
|
3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
|
|
@@ -42,7 +42,8 @@ module Saboteur
|
|
|
42
42
|
"tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
|
|
43
43
|
"directories after run replace include and exclude; --test limits the specs that run; " \
|
|
44
44
|
"--threshold PERCENT replaces the survivor check: exit 0 when the score is at or above " \
|
|
45
|
-
"PERCENT, exit 1 when it is below."
|
|
45
|
+
"PERCENT, exit 1 when it is below. Inline comment syntax: " \
|
|
46
|
+
'# saboteur:disable-line [operator ...] reason="...". A reason is required.',
|
|
46
47
|
takes_options: true,
|
|
47
48
|
exit_codes: EXIT_CODE_MEANINGS,
|
|
48
49
|
examples: [
|
|
@@ -58,14 +59,37 @@ module Saboteur
|
|
|
58
59
|
),
|
|
59
60
|
Command.new(
|
|
60
61
|
name: "doctor",
|
|
61
|
-
summary: "Check the environment and run the baseline twice, without mutating anything",
|
|
62
|
-
description: "Boots the target like a run
|
|
63
|
-
"suite twice in the same order with one seed and reports whether both
|
|
64
|
-
"It never mutates anything.",
|
|
62
|
+
summary: "Check the environment, and with --suite run the baseline twice, without mutating anything",
|
|
63
|
+
description: "Boots the target like a run and checks runtime, coverage and boot files. With --suite " \
|
|
64
|
+
"it also runs your suite twice in the same order with one seed and reports whether both " \
|
|
65
|
+
"runs passed; --test PATH limits those runs to some specs. It never mutates anything.",
|
|
65
66
|
takes_options: true,
|
|
66
67
|
exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
|
|
67
68
|
examples: [["Check the setup before the first run", "saboteur doctor"],
|
|
68
|
-
["
|
|
69
|
+
["Also run the unchanged suite twice", "saboteur doctor --suite --seed 7"],
|
|
70
|
+
["Run only some specs twice", "saboteur doctor --test spec/models"]]
|
|
71
|
+
),
|
|
72
|
+
Command.new(
|
|
73
|
+
name: "baseline",
|
|
74
|
+
summary: "Check, propose or accept a fresh opt-in survivor baseline",
|
|
75
|
+
description: "Check runs every mutant fresh and reports new, changed or unmatched accepted survivors. " \
|
|
76
|
+
"Propose writes a proposal with private local execution evidence. Edit only survivor reasons; " \
|
|
77
|
+
"accept verifies current inputs and publishes it. No scores or execution are suppressed. " \
|
|
78
|
+
"With --epsilon POINTS, check also fails when the score drops more than POINTS percentage " \
|
|
79
|
+
"points below the accepted baseline's score. " \
|
|
80
|
+
"Only whole or subject scope; no diff, replay, incremental reuse or score threshold.",
|
|
81
|
+
takes_options: true,
|
|
82
|
+
exit_codes: { 0 => "Clean check, proposal written or accepted.",
|
|
83
|
+
1 => "New, changed or unmatched survivors, or a score drop beyond --epsilon.",
|
|
84
|
+
2 => "Invalid configuration, incompatible baseline or rejected acceptance.",
|
|
85
|
+
3 => "Incomplete execution or tool failure." },
|
|
86
|
+
examples: [["Prepare a reasoned proposal",
|
|
87
|
+
"saboteur baseline propose --baseline baseline.json --proposal proposal.json"],
|
|
88
|
+
["Publish reviewed reasons",
|
|
89
|
+
"saboteur baseline accept --baseline baseline.json --proposal proposal.json"],
|
|
90
|
+
["Check fresh outcomes", "saboteur baseline check --baseline baseline.json"],
|
|
91
|
+
["Also fail on a score drop of more than 2 points",
|
|
92
|
+
"saboteur baseline check --baseline baseline.json --epsilon 2"]]
|
|
69
93
|
),
|
|
70
94
|
Command.new(
|
|
71
95
|
name: "version",
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -96,6 +96,11 @@ module Saboteur
|
|
|
96
96
|
@index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
+
# True when this result is a survivor the list accepts.
|
|
100
|
+
def accepts?(result)
|
|
101
|
+
result.survived? && @index.key?([result.subject.id, result.digest])
|
|
102
|
+
end
|
|
103
|
+
|
|
99
104
|
# Marks matching survivors as ignored. Returns warnings for stale entries:
|
|
100
105
|
# entries whose subject was examined but whose mutation no longer exists.
|
|
101
106
|
def apply(results)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require "shellwords"
|
|
6
|
+
require_relative "mutation_source/operator_registry"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
# Only real Ruby comments can suppress edits, never text inside a string.
|
|
10
|
+
class InlineSuppression
|
|
11
|
+
Directive = Data.define(:line, :operators, :reason)
|
|
12
|
+
|
|
13
|
+
def initialize(file, source)
|
|
14
|
+
@file = file
|
|
15
|
+
@directives = Prism.parse(source).comments.filter_map { |comment| parse(comment) }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Preserve ordinary comment/formatting skips, but treat directives as
|
|
19
|
+
# selection inputs. Relative lines distinguish which code they suppress.
|
|
20
|
+
def signature(subject)
|
|
21
|
+
@directives.select { |entry| subject.contains_line?(entry.line) }.map do |entry|
|
|
22
|
+
[entry.line - subject.start_line, entry.operators.sort, entry.reason]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def reason(subject, mutation)
|
|
27
|
+
mutation.edits.each do |edit|
|
|
28
|
+
line = subject.line_of(edit.start_byte)
|
|
29
|
+
directive = @directives.find do |entry|
|
|
30
|
+
entry.line == line && (entry.operators.empty? || entry.operators.include?(mutation.operator))
|
|
31
|
+
end
|
|
32
|
+
return directive if directive
|
|
33
|
+
end
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def parse(comment)
|
|
40
|
+
text = comment.location.slice.sub(/\A#\s*/, "")
|
|
41
|
+
return nil unless text.start_with?("saboteur:disable-line")
|
|
42
|
+
|
|
43
|
+
line = comment.location.start_line
|
|
44
|
+
tokens = Shellwords.split(text)
|
|
45
|
+
fail_at(line, "expected saboteur:disable-line") unless tokens.shift == "saboteur:disable-line"
|
|
46
|
+
reasons, operators = tokens.partition { |token| token.start_with?("reason=") }
|
|
47
|
+
reason, = reasons
|
|
48
|
+
fail_at(line, "one nonblank reason=\"...\" is required") unless
|
|
49
|
+
reasons.size == 1 && !reason.delete_prefix("reason=").strip.empty?
|
|
50
|
+
validate_operators(operators, line)
|
|
51
|
+
Directive.new(line: line, operators: operators, reason: reason.delete_prefix("reason="))
|
|
52
|
+
rescue ArgumentError => e
|
|
53
|
+
fail_at(line, "invalid suppression: #{e.message}")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def validate_operators(operators, line)
|
|
57
|
+
unknown = operators - MutationSource::OperatorRegistry::DESCRIPTIONS.keys
|
|
58
|
+
fail_at(line, "unknown operators: #{unknown.join(', ')}") unless unknown.empty?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def fail_at(line, message)
|
|
62
|
+
raise ConfigError, "#{@file}:#{line}: #{message}"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Integration
|
|
6
|
+
# Refuses a suite that a focus filter has narrowed without the user asking
|
|
7
|
+
# for it. With `filter_run_when_matching :focus`, one stray `fit` makes
|
|
8
|
+
# RSpec skip every other example. The baseline stays green and the score
|
|
9
|
+
# then describes only the focused examples, with no sign that the rest
|
|
10
|
+
# never ran.
|
|
11
|
+
#
|
|
12
|
+
# The test is a difference of two selections, both made by RSpec's own
|
|
13
|
+
# FilterManager#prune: the examples that run now, and the examples that
|
|
14
|
+
# would run if the stray focus rule were absent. Everything else RSpec
|
|
15
|
+
# decides stays the same in both, so it never counts as hidden: example
|
|
16
|
+
# selectors (`spec/a_spec.rb:12`) and their file-scoped priority,
|
|
17
|
+
# `--tag` filters named in rspec_args, exclusions, and the :if / :unless
|
|
18
|
+
# pruning of conditional examples.
|
|
19
|
+
class FocusGuard
|
|
20
|
+
LOCATION_KEYS = %i[locations ids].freeze
|
|
21
|
+
|
|
22
|
+
def initialize(world:, explicit_filter_keys:)
|
|
23
|
+
@world = world
|
|
24
|
+
@explicit_filter_keys = explicit_filter_keys
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Raises ConfigError naming the focused examples when the focus filter
|
|
28
|
+
# hides examples that would otherwise run.
|
|
29
|
+
def check!
|
|
30
|
+
return if stray_rules.empty?
|
|
31
|
+
return if hidden_examples.empty?
|
|
32
|
+
|
|
33
|
+
raise ConfigError, message
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
attr_reader :world
|
|
39
|
+
|
|
40
|
+
def filter_manager
|
|
41
|
+
world.filter_manager
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Inclusion rules the suite added itself, such as the :focus rule that
|
|
45
|
+
# `filter_run_when_matching :focus` installs once a fit exists.
|
|
46
|
+
def stray_rules
|
|
47
|
+
@stray_rules ||= begin
|
|
48
|
+
rules = filter_manager.inclusions.rules
|
|
49
|
+
rules.slice(*(rules.keys - LOCATION_KEYS - @explicit_filter_keys))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def running_examples
|
|
54
|
+
@running_examples ||= filter_manager.prune(world.all_examples)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The selection RSpec would make without the stray rules.
|
|
58
|
+
def examples_without_stray_focus
|
|
59
|
+
manager = ::RSpec::Core::FilterManager.new
|
|
60
|
+
manager.exclusions.rules.replace(filter_manager.exclusions.rules)
|
|
61
|
+
manager.inclusions.rules.replace(filter_manager.inclusions.rules.except(*stray_rules.keys))
|
|
62
|
+
manager.prune(world.all_examples)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def hidden_examples
|
|
66
|
+
@hidden_examples ||= examples_without_stray_focus - running_examples
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def focused_examples
|
|
70
|
+
focus = ::RSpec::Core::FilterRules.new(stray_rules)
|
|
71
|
+
running_examples.select { |example| focus.include_example?(example) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def message
|
|
75
|
+
total = hidden_examples.size + running_examples.size
|
|
76
|
+
locations = focused_examples.map(&:location).uniq.first(5).join(", ")
|
|
77
|
+
"a focus filter (#{::RSpec::Core::FilterRules.new(stray_rules).description}) narrowed the suite: " \
|
|
78
|
+
"#{hidden_examples.size} of #{total} examples would not run, so the score would cover only the " \
|
|
79
|
+
"ones that do (#{locations}). Remove the focus, or pass --allow-focus to run the narrowed suite anyway."
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -14,6 +14,8 @@ module Saboteur
|
|
|
14
14
|
@runner.setup($stderr, $stdout)
|
|
15
15
|
configuration.order = "rand:#{config.fetch('seed', 1)}"
|
|
16
16
|
raise Error, "rspec quit while loading spec files" if world.wants_to_quit
|
|
17
|
+
|
|
18
|
+
refuse_unintended_focus
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
# RSpec only adds its default "spec" path when $0 is `rspec`.
|
|
@@ -48,7 +50,7 @@ module Saboteur
|
|
|
48
50
|
wanted = Set.new(ids)
|
|
49
51
|
narrow_to(wanted)
|
|
50
52
|
order_by(ids)
|
|
51
|
-
configuration.fail_fast = 1
|
|
53
|
+
configuration.fail_fast = config["matrix"] ? false : 1
|
|
52
54
|
listener = Listener.new
|
|
53
55
|
passed = execute(listener)
|
|
54
56
|
RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
|
|
@@ -74,6 +76,24 @@ module Saboteur
|
|
|
74
76
|
|
|
75
77
|
private
|
|
76
78
|
|
|
79
|
+
def refuse_unintended_focus
|
|
80
|
+
return if config["allow_focus"]
|
|
81
|
+
|
|
82
|
+
FocusGuard.new(world:, explicit_filter_keys: explicit_inclusion_keys).check!
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Inclusion filters the user named in rspec_args or .rspec, such as
|
|
86
|
+
# `--tag slow`. Those narrow the run on purpose.
|
|
87
|
+
def explicit_inclusion_keys
|
|
88
|
+
options = configuration_options
|
|
89
|
+
filter_manager = ::RSpec::Core::FilterManager.new
|
|
90
|
+
options.configure_filter_manager(filter_manager)
|
|
91
|
+
keys = filter_manager.inclusions.rules.keys
|
|
92
|
+
# `--example TEXT` becomes a :full_description rule when the options
|
|
93
|
+
# reach the configuration, not in the filter manager.
|
|
94
|
+
options.options[:full_description] ? keys + [:full_description] : keys
|
|
95
|
+
end
|
|
96
|
+
|
|
77
97
|
def world
|
|
78
98
|
::RSpec.world
|
|
79
99
|
end
|
|
@@ -6,7 +6,13 @@ require_relative "base"
|
|
|
6
6
|
require_relative "filter_replacement"
|
|
7
7
|
require_relative "literal_replacement"
|
|
8
8
|
require_relative "argument_removal"
|
|
9
|
+
require_relative "chain_removal"
|
|
9
10
|
require_relative "pair_removal"
|
|
11
|
+
require_relative "endpoint_replacement"
|
|
12
|
+
require_relative "operator_registry"
|
|
13
|
+
require_relative "range_replacement"
|
|
14
|
+
require_relative "operator_assignment"
|
|
15
|
+
require_relative "condition_negation"
|
|
10
16
|
|
|
11
17
|
module Saboteur
|
|
12
18
|
module MutationSource
|
|
@@ -15,7 +21,7 @@ module Saboteur
|
|
|
15
21
|
class Builtin < Base
|
|
16
22
|
DEFAULT_ARID = [
|
|
17
23
|
'\A(puts|pp|p|print|warn)\b',
|
|
18
|
-
'\A(
|
|
24
|
+
'\A(@|self\.|::)?(Rails\.)?logger\b',
|
|
19
25
|
'\A(::)?(StatsD|Statsd|Datadog|NewRelic)\b',
|
|
20
26
|
'\.instrument\b'
|
|
21
27
|
].freeze
|
|
@@ -47,13 +53,19 @@ module Saboteur
|
|
|
47
53
|
CATCH_ALL = %w[StandardError ::StandardError Exception ::Exception].freeze
|
|
48
54
|
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
49
55
|
|
|
50
|
-
def initialize(arid: DEFAULT_ARID)
|
|
56
|
+
def initialize(arid: DEFAULT_ARID, operators: [])
|
|
51
57
|
super()
|
|
58
|
+
@operators = OperatorRegistry.resolve(operators)
|
|
52
59
|
@arid = arid.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) }
|
|
53
60
|
@filter_replacement = FilterReplacement.new(arid: arid)
|
|
54
61
|
@literal_replacement = LiteralReplacement.new(arid: arid)
|
|
55
62
|
@argument_removal = ArgumentRemoval.new(arid: arid)
|
|
56
63
|
@pair_removal = PairRemoval.new(arid: arid)
|
|
64
|
+
@chain_removal = ChainRemoval.new(arid: arid)
|
|
65
|
+
@endpoint_replacement = EndpointReplacement.new(arid: arid)
|
|
66
|
+
@range_replacement = RangeReplacement.new(arid: arid)
|
|
67
|
+
@operator_assignment = OperatorAssignment.new(arid: arid)
|
|
68
|
+
@condition_negation = ConditionNegation.new(arid: arid)
|
|
57
69
|
end
|
|
58
70
|
|
|
59
71
|
def name
|
|
@@ -70,6 +82,12 @@ module Saboteur
|
|
|
70
82
|
@found.concat(@literal_replacement.mutations_for(subject))
|
|
71
83
|
@found.concat(@argument_removal.mutations_for(subject))
|
|
72
84
|
@found.concat(@pair_removal.mutations_for(subject))
|
|
85
|
+
@found.concat(@chain_removal.mutations_for(subject))
|
|
86
|
+
@found.concat(@endpoint_replacement.mutations_for(subject))
|
|
87
|
+
@found.concat(@range_replacement.mutations_for(subject))
|
|
88
|
+
@found.concat(@operator_assignment.mutations_for(subject))
|
|
89
|
+
@found.concat(@condition_negation.mutations_for(subject))
|
|
90
|
+
@found.select { |mutation| @operators.include?(mutation.operator) }
|
|
73
91
|
ensure
|
|
74
92
|
@subject = nil
|
|
75
93
|
end
|
|
@@ -134,7 +152,8 @@ module Saboteur
|
|
|
134
152
|
statements.each do |statement|
|
|
135
153
|
next if arid?(statement)
|
|
136
154
|
|
|
137
|
-
|
|
155
|
+
removable_here = statements.drop(1).any? && removable?(statement)
|
|
156
|
+
add("statement_removal", statement, "nil") if removable_here
|
|
138
157
|
visit(statement)
|
|
139
158
|
end
|
|
140
159
|
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Drops one link from the middle of a method chain: a.b.c becomes a.c.
|
|
10
|
+
# The first receiver and the last call stay, so the result is still a call
|
|
11
|
+
# on the original object.
|
|
12
|
+
class ChainRemoval < Base
|
|
13
|
+
BOUNDARIES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
14
|
+
# Links this operator leaves alone by design. They hand back their receiver or a
|
|
15
|
+
# copy of it (tap after running a block), so dropping one changes mutability,
|
|
16
|
+
# copying or a block's side effects. Tests can observe those, but they are other
|
|
17
|
+
# contracts than "a step in the query chain went missing", and removing them
|
|
18
|
+
# mostly yields noise, so they are out of scope here.
|
|
19
|
+
IDENTITY_CALLS = %w[freeze dup clone itself tap].freeze
|
|
20
|
+
|
|
21
|
+
# Targets of a write: `a.b.c = 1`, `a.b.c += 1`, `a.b[0] ||= 1`. The whole receiver chain
|
|
22
|
+
# is the thing being assigned to, so no link of it is dropped.
|
|
23
|
+
WRITE_NODES = [Prism::CallOperatorWriteNode, Prism::CallOrWriteNode, Prism::CallAndWriteNode,
|
|
24
|
+
Prism::IndexOperatorWriteNode, Prism::IndexOrWriteNode, Prism::IndexAndWriteNode].freeze
|
|
25
|
+
|
|
26
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
27
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def name
|
|
31
|
+
"chain_removal"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def mutations_for(subject)
|
|
35
|
+
found = []
|
|
36
|
+
visit(subject.node.body, subject, found)
|
|
37
|
+
found
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def visit(node, subject, found)
|
|
43
|
+
return if node.nil? || BOUNDARIES.any? { |type| node.is_a?(type) }
|
|
44
|
+
|
|
45
|
+
if node.is_a?(Prism::StatementsNode)
|
|
46
|
+
node.body.each { |child| visit(child, subject, found) unless @arid.any? { |p| p.match?(child.slice) } }
|
|
47
|
+
return
|
|
48
|
+
end
|
|
49
|
+
remove_link(node, subject, found) if node.is_a?(Prism::CallNode)
|
|
50
|
+
children(node).each { |child| visit(child, subject, found) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Everything under the node except the receiver chain of a write target.
|
|
54
|
+
def children(node)
|
|
55
|
+
return node.compact_child_nodes unless write_target?(node)
|
|
56
|
+
|
|
57
|
+
node.compact_child_nodes.reject { |child| child.equal?(node.receiver) }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def write_target?(node)
|
|
61
|
+
WRITE_NODES.any? { |type| node.is_a?(type) } || (node.is_a?(Prism::CallNode) && node.attribute_write?)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def remove_link(call, subject, found)
|
|
65
|
+
link = call.receiver
|
|
66
|
+
return unless call.call_operator_loc && !call.attribute_write? && removable?(link)
|
|
67
|
+
|
|
68
|
+
edit = Edit.new(file: subject.file, start_byte: link.receiver.end_offset,
|
|
69
|
+
end_byte: link.end_offset, replacement: "")
|
|
70
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# A dotted call with a method name (a dot always has a receiver).
|
|
74
|
+
def removable?(link)
|
|
75
|
+
link.is_a?(Prism::CallNode) && link.call_operator_loc && link.message_loc &&
|
|
76
|
+
!IDENTITY_CALLS.include?(link.name.to_s)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "base"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module MutationSource
|
|
8
|
+
# Negates the predicate, retaining its evaluation and branch structure.
|
|
9
|
+
class ConditionNegation < Base
|
|
10
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
11
|
+
|
|
12
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
13
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def name
|
|
17
|
+
"condition_negation"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def mutations_for(subject)
|
|
21
|
+
found = []
|
|
22
|
+
visit(subject.node.body, subject, found)
|
|
23
|
+
found
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def visit(node, subject, found)
|
|
29
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
30
|
+
return if @arid.any? { |pattern| pattern.match?(node.slice) }
|
|
31
|
+
|
|
32
|
+
negate(node.predicate, subject, found) if node.is_a?(Prism::IfNode) || node.is_a?(Prism::UnlessNode)
|
|
33
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def negate(predicate, subject, found)
|
|
37
|
+
# Existing true/false and negation-removal operators own these cases.
|
|
38
|
+
if predicate.is_a?(Prism::TrueNode) || predicate.is_a?(Prism::FalseNode) || predicate.is_a?(Prism::NilNode)
|
|
39
|
+
return
|
|
40
|
+
end
|
|
41
|
+
return if predicate.is_a?(Prism::CallNode) && predicate.name == :!
|
|
42
|
+
|
|
43
|
+
location = predicate.location
|
|
44
|
+
edit = Edit.new(file: subject.file, start_byte: location.start_offset,
|
|
45
|
+
end_byte: location.end_offset, replacement: "!(#{predicate.slice})")
|
|
46
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|