saboteur 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +167 -0
- data/README.md +393 -20
- data/exe/saboteur +6 -2
- data/lib/saboteur/agent_skill.rb +6 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +105 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +114 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +101 -121
- data/lib/saboteur/config.rb +76 -6
- data/lib/saboteur/doctor.rb +40 -8
- data/lib/saboteur/engine/baseline_guard.rb +33 -11
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +59 -7
- data/lib/saboteur/engine/coverage_store.rb +34 -14
- data/lib/saboteur/engine/diagnostics.rb +150 -4
- data/lib/saboteur/engine/diff_scope.rb +10 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +6 -4
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +92 -20
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/test_guess.rb +52 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +192 -18
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +72 -10
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +101 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +31 -7
- data/lib/saboteur/ignore_list.rb +5 -0
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +21 -1
- data/lib/saboteur/mutation_source/builtin.rb +22 -3
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +106 -0
- data/lib/saboteur/reporter/agent_commands.rb +7 -1
- data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
- data/lib/saboteur/reporter/agent_summary.rb +11 -5
- data/lib/saboteur/reporter/findings.rb +4 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_record.rb +132 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +123 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +15 -2
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +131 -79
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +41 -6
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +8 -0
- data/lib/saboteur.rb +6 -0
- metadata +43 -2
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Swaps the selected end of an explicitly received collection.
|
|
10
|
+
# Only message bytes change; arguments, receiver and block stay intact.
|
|
11
|
+
class EndpointReplacement < Base
|
|
12
|
+
REPLACEMENTS = { first: "last", last: "first" }.freeze
|
|
13
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
14
|
+
|
|
15
|
+
# Base inherits Object's empty initializer; there is no parent state to set.
|
|
16
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
17
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def name
|
|
21
|
+
"endpoint_replacement"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def mutations_for(subject)
|
|
25
|
+
found = []
|
|
26
|
+
visit(subject.node.body, subject, found)
|
|
27
|
+
found
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def visit(node, subject, found)
|
|
33
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
34
|
+
|
|
35
|
+
if node.is_a?(Prism::StatementsNode)
|
|
36
|
+
node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
|
|
37
|
+
return
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
replace_endpoint(node, subject, found) if node.is_a?(Prism::CallNode)
|
|
41
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def replace_endpoint(node, subject, found)
|
|
45
|
+
replacement = REPLACEMENTS[node.name]
|
|
46
|
+
return unless replacement && node.receiver
|
|
47
|
+
return if node.block
|
|
48
|
+
return unless node.arguments.nil?
|
|
49
|
+
|
|
50
|
+
location = node.message_loc
|
|
51
|
+
edit = Edit.new(file: subject.file, start_byte: location.start_offset,
|
|
52
|
+
end_byte: location.end_offset, replacement: replacement)
|
|
53
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def arid?(node)
|
|
57
|
+
@arid.any? { |pattern| pattern.match?(node.slice) }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module MutationSource
|
|
6
|
+
# Suppresses mutants that would run a destructive call with altered
|
|
7
|
+
# inputs. A mutant is only kept when every edit is provably safe: either
|
|
8
|
+
# the subject holds no hazardous call, or the edit removes every
|
|
9
|
+
# hazardous call in the method AND its replacement introduces no new one
|
|
10
|
+
# (checked by parsing the replacement, never by substring match). Edits
|
|
11
|
+
# strictly inside a call, edits elsewhere in a method that contains a
|
|
12
|
+
# call, and edits whose replacement re-includes a hazard (for example
|
|
13
|
+
# removing a negation in front of `system(...)`, or swapping
|
|
14
|
+
# `rm_rf("build")` for `rm_rf(".")`) all leave a destructive call
|
|
15
|
+
# running with altered inputs and are suppressed (issue #102).
|
|
16
|
+
class HazardGuard
|
|
17
|
+
# "<Receiver>.<name>" for calls with a receiver, the bare name for
|
|
18
|
+
# Kernel-style calls. Bare FileUtils operations are listed too, so an
|
|
19
|
+
# `include FileUtils` does not disarm the guard. Any method on Open3,
|
|
20
|
+
# any rmtree, and backticks/%x literals are always hazardous. Replace
|
|
21
|
+
# or extend via c.hazard; run them anyway with --allow-hazardous.
|
|
22
|
+
DEFAULT_HAZARD = [
|
|
23
|
+
"FileUtils.rm", "FileUtils.rm_r", "FileUtils.rm_rf", "FileUtils.rm_f",
|
|
24
|
+
"FileUtils.remove_dir", "FileUtils.remove_entry", "FileUtils.remove_entry_secure",
|
|
25
|
+
"rm", "rm_r", "rm_rf", "rm_f", "remove_dir", "remove_entry", "remove_entry_secure",
|
|
26
|
+
"File.delete", "File.unlink",
|
|
27
|
+
"Dir.rmdir", "Dir.delete", "Dir.unlink",
|
|
28
|
+
"IO.popen",
|
|
29
|
+
"Process.spawn", "Process.exec",
|
|
30
|
+
"system", "exec", "spawn", "`"
|
|
31
|
+
].freeze
|
|
32
|
+
|
|
33
|
+
OPEN3 = /\A(::)?Open3\z/
|
|
34
|
+
ALWAYS_NAMES = %w[rmtree].freeze
|
|
35
|
+
KERNEL_RECEIVERS = %w[Kernel Process].freeze
|
|
36
|
+
|
|
37
|
+
def initialize(patterns = DEFAULT_HAZARD)
|
|
38
|
+
@tags = Set.new(patterns.map { |pattern| pattern.to_s.sub(/\A::/, "") })
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def name
|
|
42
|
+
"hazard_guard"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# candidates: [[mutation, mutated_text]]. Returns [kept, suppressed].
|
|
46
|
+
def partition(candidates, subject)
|
|
47
|
+
spans = hazard_spans(subject.node)
|
|
48
|
+
return [candidates, []] if spans.empty?
|
|
49
|
+
|
|
50
|
+
candidates.partition { |mutation, _| safe?(mutation, spans) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
# Byte spans of hazardous calls in the node tree, with the source slice
|
|
56
|
+
# of each. Inner calls are covered by the outer span, so the walk does
|
|
57
|
+
# not descend into a hazardous call.
|
|
58
|
+
def hazard_spans(root)
|
|
59
|
+
spans = []
|
|
60
|
+
stack = [root]
|
|
61
|
+
until stack.empty?
|
|
62
|
+
node = stack.pop
|
|
63
|
+
if hazardous?(node)
|
|
64
|
+
spans << [node.location.start_offset, node.location.end_offset, node.slice]
|
|
65
|
+
next
|
|
66
|
+
end
|
|
67
|
+
stack.concat(node.compact_child_nodes)
|
|
68
|
+
end
|
|
69
|
+
spans
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def hazardous?(node)
|
|
73
|
+
case node
|
|
74
|
+
when Prism::XStringNode, Prism::InterpolatedXStringNode then true
|
|
75
|
+
when Prism::CallNode then hazardous_call?(node)
|
|
76
|
+
else false
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def hazardous_call?(node)
|
|
81
|
+
name = node.name.to_s
|
|
82
|
+
receiver = node.receiver&.slice&.sub(/\A::/, "")
|
|
83
|
+
return true if OPEN3.match?(receiver.to_s)
|
|
84
|
+
return true if ALWAYS_NAMES.include?(name)
|
|
85
|
+
|
|
86
|
+
tag = receiver ? "#{receiver}.#{name}" : name
|
|
87
|
+
return true if @tags.include?(tag)
|
|
88
|
+
# Kernel.system / Kernel.spawn / Process.spawn: the explicit-receiver
|
|
89
|
+
# spelling of a bare process spawn is the same hazard.
|
|
90
|
+
return true if KERNEL_RECEIVERS.include?(receiver) && @tags.include?(name)
|
|
91
|
+
|
|
92
|
+
piped_open?(node, name, receiver)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Kernel#open with a "| command" argument spawns a process.
|
|
96
|
+
def piped_open?(node, name, receiver)
|
|
97
|
+
return false unless name == "open" && (receiver.nil? || receiver == "Kernel")
|
|
98
|
+
|
|
99
|
+
first = node.arguments&.arguments&.first
|
|
100
|
+
case first
|
|
101
|
+
when Prism::StringNode then first.unescaped.start_with?("|")
|
|
102
|
+
when Prism::InterpolatedStringNode
|
|
103
|
+
part = first.parts.first
|
|
104
|
+
part.is_a?(Prism::StringNode) && part.unescaped.start_with?("|")
|
|
105
|
+
else false
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# A mutation is safe only while no hazardous call survives it: every
|
|
110
|
+
# edit must destroy every span (cover it wholly with a hazard-free
|
|
111
|
+
# replacement). Anything less - touching a call, or editing other
|
|
112
|
+
# code in a method that still holds a call - keeps the call running
|
|
113
|
+
# with altered inputs, so the mutant is suppressed.
|
|
114
|
+
def safe?(mutation, spans)
|
|
115
|
+
mutation.edits.all? { |edit| destroys_all?(edit, spans) }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def destroys_all?(edit, spans)
|
|
119
|
+
spans.all? do |start_byte, end_byte, _slice|
|
|
120
|
+
edit.start_byte <= start_byte && edit.end_byte >= end_byte
|
|
121
|
+
end && replacement_clean?(edit.replacement)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The replacement itself, parsed as code. A replacement that parses
|
|
125
|
+
# and contains no hazardous call cannot run one; anything else -
|
|
126
|
+
# a hazard in the new text, or text that does not parse on its own -
|
|
127
|
+
# is treated as hazardous.
|
|
128
|
+
def replacement_clean?(replacement)
|
|
129
|
+
result = Prism.parse(replacement.to_s)
|
|
130
|
+
return false unless result.success?
|
|
131
|
+
|
|
132
|
+
hazard_spans(result.value).empty?
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "base"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module MutationSource
|
|
8
|
+
# Changes the arithmetic operation, without changing the assignment target.
|
|
9
|
+
class OperatorAssignment < Base
|
|
10
|
+
REPLACEMENTS = { :+ => "-=", :- => "+=", :* => "/=", :/ => "*=" }.freeze
|
|
11
|
+
WRITE_NODES = [Prism::LocalVariableOperatorWriteNode, Prism::InstanceVariableOperatorWriteNode,
|
|
12
|
+
Prism::ClassVariableOperatorWriteNode, Prism::GlobalVariableOperatorWriteNode,
|
|
13
|
+
Prism::ConstantOperatorWriteNode, Prism::ConstantPathOperatorWriteNode,
|
|
14
|
+
Prism::CallOperatorWriteNode, Prism::IndexOperatorWriteNode].freeze
|
|
15
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
16
|
+
|
|
17
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
18
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def name
|
|
22
|
+
"operator_assignment"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def mutations_for(subject)
|
|
26
|
+
found = []
|
|
27
|
+
visit(subject.node.body, subject, found)
|
|
28
|
+
found
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def visit(node, subject, found)
|
|
34
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
35
|
+
return if @arid.any? { |pattern| pattern.match?(node.slice) }
|
|
36
|
+
|
|
37
|
+
replace(node, subject, found) if WRITE_NODES.any? { |type| node.is_a?(type) }
|
|
38
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def replace(node, subject, found)
|
|
42
|
+
replacement = REPLACEMENTS[node.binary_operator]
|
|
43
|
+
return unless replacement
|
|
44
|
+
|
|
45
|
+
location = node.binary_operator_loc
|
|
46
|
+
edit = Edit.new(file: subject.file, start_byte: location.start_offset,
|
|
47
|
+
end_byte: location.end_offset, replacement: replacement)
|
|
48
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module MutationSource
|
|
6
|
+
# Public names used by selection, help and inline suppression validation.
|
|
7
|
+
module OperatorRegistry
|
|
8
|
+
DESCRIPTIONS = {
|
|
9
|
+
"operator_assignment" => "Swap arithmetic compound assignments",
|
|
10
|
+
"condition_negation" => "Negate if, unless and ternary predicates",
|
|
11
|
+
"range" => "Swap inclusive and exclusive bounded ranges",
|
|
12
|
+
"body_nil" => "Replace the method body with nil",
|
|
13
|
+
"body_true" => "Replace a predicate body with true",
|
|
14
|
+
"body_false" => "Replace a predicate body with false",
|
|
15
|
+
"body_raise" => "Replace the method body with raise",
|
|
16
|
+
"comparison" => "Change a comparison or replace its result",
|
|
17
|
+
"arithmetic" => "Swap an arithmetic operator",
|
|
18
|
+
"boolean_logic" => "Change a boolean connector",
|
|
19
|
+
"condition_true" => "Force a condition true",
|
|
20
|
+
"condition_false" => "Force a condition false",
|
|
21
|
+
"negation_removal" => "Remove a logical negation",
|
|
22
|
+
"literal_number" => "Change a numeric literal",
|
|
23
|
+
"or_assign" => "Replace ||= with assignment",
|
|
24
|
+
"safe_navigation" => "Replace safe navigation with a regular call",
|
|
25
|
+
"statement_removal" => "Replace a statement with nil",
|
|
26
|
+
"filter_replacement" => "Swap select and reject",
|
|
27
|
+
"literal_string" => "Empty a string literal",
|
|
28
|
+
"literal_boolean" => "Swap a boolean literal",
|
|
29
|
+
"argument_removal" => "Remove a call argument",
|
|
30
|
+
"pair_removal" => "Remove a hash pair",
|
|
31
|
+
"endpoint_replacement" => "Swap explicit no-argument first and last",
|
|
32
|
+
"chain_removal" => "Remove one call from a method chain"
|
|
33
|
+
}.freeze
|
|
34
|
+
EXTENDED = %w[range operator_assignment condition_negation].freeze
|
|
35
|
+
TIERS = { "core" => (DESCRIPTIONS.keys - EXTENDED).freeze, "extended" => EXTENDED,
|
|
36
|
+
"all" => DESCRIPTIONS.keys.freeze }.freeze
|
|
37
|
+
|
|
38
|
+
def self.list(out)
|
|
39
|
+
DESCRIPTIONS.each do |name, description|
|
|
40
|
+
out.puts("#{name} #{EXTENDED.include?(name) ? 'extended' : 'core'} #{description}")
|
|
41
|
+
end
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.resolve(names)
|
|
46
|
+
return TIERS.fetch("core") if names.empty?
|
|
47
|
+
|
|
48
|
+
unknown = names - DESCRIPTIONS.keys - TIERS.keys
|
|
49
|
+
unless unknown.empty?
|
|
50
|
+
raise ConfigError, "unknown operators: #{unknown.join(', ')}. Valid names: " \
|
|
51
|
+
"#{(DESCRIPTIONS.keys + TIERS.keys).join(', ')}"
|
|
52
|
+
end
|
|
53
|
+
names.flat_map { |name| TIERS.fetch(name) { [name] } }.uniq
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "base"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module MutationSource
|
|
8
|
+
# Changes whether a range includes its upper endpoint.
|
|
9
|
+
class RangeReplacement < Base
|
|
10
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
11
|
+
|
|
12
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
13
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def name
|
|
17
|
+
"range"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def mutations_for(subject)
|
|
21
|
+
found = []
|
|
22
|
+
visit(subject.node.body, subject, found)
|
|
23
|
+
found
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def visit(node, subject, found)
|
|
29
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
30
|
+
return if @arid.any? { |pattern| pattern.match?(node.slice) }
|
|
31
|
+
|
|
32
|
+
if node.is_a?(Prism::RangeNode) && node.left && node.right
|
|
33
|
+
location = node.operator_loc
|
|
34
|
+
replacement = location.slice == ".." ? "..." : ".."
|
|
35
|
+
edit = Edit.new(file: subject.file, start_byte: location.start_offset,
|
|
36
|
+
end_byte: location.end_offset, replacement: replacement)
|
|
37
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
38
|
+
end
|
|
39
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "engine"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
# Source-only planning. Never boots the host or executes its test suite.
|
|
8
|
+
class Preview
|
|
9
|
+
def initialize(config, out:, err:)
|
|
10
|
+
@config = config
|
|
11
|
+
@out = out
|
|
12
|
+
@diagnostics = Engine::Diagnostics.new(err)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call
|
|
16
|
+
selection = Engine::SubjectSelection.new(@config, @diagnostics)
|
|
17
|
+
subjects = selection.select(@config.included_files)
|
|
18
|
+
scope = selection.changed_lines && Engine::DiffScope.new(selection.changed_lines)
|
|
19
|
+
guard = MutationSource::HazardGuard.new(@config["hazard"]) unless @config["allow_hazardous"]
|
|
20
|
+
candidates = Engine::Candidates.new(Engine::Candidates.build_sources(@config), @diagnostics, hazard_guard: guard)
|
|
21
|
+
lists = subjects.map { |subject| planned(subject, candidates, scope) }
|
|
22
|
+
inline_suppressions
|
|
23
|
+
incomplete = @diagnostics.discovery_failures.size
|
|
24
|
+
@out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
|
|
25
|
+
"dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
|
|
26
|
+
incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
|
|
27
|
+
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
28
|
+
nil
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
34
|
+
# Name that scope rather than implying these rows were executed or scored.
|
|
35
|
+
def inline_suppressions
|
|
36
|
+
entries = @diagnostics.inline_suppressions
|
|
37
|
+
return if entries.empty?
|
|
38
|
+
|
|
39
|
+
@out.puts("Inline suppressions: #{entries.size} candidates in selected methods " \
|
|
40
|
+
"(before diff restriction and per-method cap; outside the planned total)")
|
|
41
|
+
entries.each do |entry|
|
|
42
|
+
path = entry[:file].delete_prefix("#{@config.root}/")
|
|
43
|
+
@out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
44
|
+
"#{entry[:digest]}: #{entry[:reason]}")
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def incomplete_note(count)
|
|
49
|
+
return "" if count.zero?
|
|
50
|
+
|
|
51
|
+
" - INCOMPLETE: #{count} file(s) could not be parsed, so this total is not authoritative"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def estimate(subjects, lists)
|
|
55
|
+
map = cached_map
|
|
56
|
+
return @out.puts("no estimate: run once to build the coverage map") unless map
|
|
57
|
+
|
|
58
|
+
seconds = work_seconds(map, subjects, lists)
|
|
59
|
+
return @out.puts("no estimate: cached test durations are incomplete") unless seconds
|
|
60
|
+
|
|
61
|
+
@out.puts("Estimated baseline covering-test work: #{seconds.round(2)} s (cached original-test durations " \
|
|
62
|
+
"times planned mutants; not a bound on mutant runtime; excludes boot and forks)")
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Cached data is untrusted input: every duration must be a non-negative
|
|
66
|
+
# number, the total must stay finite (JSON text such as 1e999 parses to
|
|
67
|
+
# Infinity), and anything that fails while reading it, such as a string or
|
|
68
|
+
# null, is no estimate. Source generation has already finished, so this
|
|
69
|
+
# cannot hide its errors.
|
|
70
|
+
def work_seconds(map, subjects, lists)
|
|
71
|
+
return unless map.durations.values.all? { |seconds| seconds >= 0 }
|
|
72
|
+
|
|
73
|
+
total = subjects.zip(lists).sum do |subject, list|
|
|
74
|
+
# nil when a covering test has no recorded duration: the rescue below.
|
|
75
|
+
map.recorded_runtime(map.tests_for(subject)) * list.size
|
|
76
|
+
end
|
|
77
|
+
total if total.finite?
|
|
78
|
+
rescue StandardError
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Any receipt that is not exactly a current, readable pointer to a
|
|
83
|
+
# coverage map means "no estimate", never an exception.
|
|
84
|
+
def cached_map
|
|
85
|
+
path = File.join(@config.path("cache_dir"), "preview.json")
|
|
86
|
+
receipt = JSON.parse(File.read(path))
|
|
87
|
+
return unless receipt["fingerprint"] == Engine::RunFingerprint.new(config: @config).snapshot.digest
|
|
88
|
+
|
|
89
|
+
Selection::CoverageMap.load(receipt["path"])
|
|
90
|
+
rescue StandardError
|
|
91
|
+
# An unreadable receipt or a map of the wrong shape (IndexError, TypeError, ...) is no estimate.
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def planned(subject, candidates, scope)
|
|
96
|
+
list = candidates.for(subject)
|
|
97
|
+
list = scope.restrict(subject, list) if scope
|
|
98
|
+
limit = @config["max_mutants_per_method"]
|
|
99
|
+
@diagnostics.record_omission(subject, count: list.size - limit, limit: limit) if list.size > limit
|
|
100
|
+
list = list.first(limit)
|
|
101
|
+
path = subject.file.delete_prefix("#{@config.root}/")
|
|
102
|
+
@out.puts("#{path}:#{subject.start_line} #{subject.id} #{list.size} mutants")
|
|
103
|
+
list
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -40,12 +40,18 @@ module Saboteur
|
|
|
40
40
|
command.bytesize <= MAX_BYTES ? command : nil
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
+
# An explicit scope is frozen: --no-guess-tests keeps a config-file
|
|
44
|
+
# guess_tests from conflicting with the --test list.
|
|
45
|
+
def test_words(tests)
|
|
46
|
+
tests.empty? ? [] : ["--no-guess-tests", *tests.flat_map { |test| ["--test", test] }]
|
|
47
|
+
end
|
|
48
|
+
|
|
43
49
|
def selection(provenance, replay: false)
|
|
44
50
|
words = []
|
|
45
51
|
words.push("--config", provenance.config_path) if provenance.config_path
|
|
46
52
|
words.push("--since", provenance.ref) if provenance.ref
|
|
47
53
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
48
|
-
|
|
54
|
+
words.push(*test_words(provenance.tests))
|
|
49
55
|
SETTING_FLAGS.each do |key, flag|
|
|
50
56
|
words.push(flag, provenance.settings[key].to_s) if provenance.settings.key?(key)
|
|
51
57
|
end
|
|
@@ -7,24 +7,47 @@ module Saboteur
|
|
|
7
7
|
# What makes a verdict cover less than the whole selection. Each reason has a
|
|
8
8
|
# stable code, a count and a plain meaning.
|
|
9
9
|
module Incomplete
|
|
10
|
+
NO_COVERAGE = "no coverage"
|
|
11
|
+
|
|
10
12
|
class << self
|
|
11
13
|
def reasons(report)
|
|
12
14
|
reasons = []
|
|
13
15
|
add(reasons, "no_coverage", skipped_for_coverage(report), "subjects or mutants had no covering test")
|
|
16
|
+
load_only = report.counts[:ran_at_load] + report.skipped_subjects.values.count do |reason|
|
|
17
|
+
reason == Selection::CoverageMap::RAN_AT_LOAD
|
|
18
|
+
end
|
|
19
|
+
add(reasons, "unsupported", unsupported_subjects(report),
|
|
20
|
+
"subjects were skipped as unsupported (for example defined inside a block); no mutant was judged")
|
|
21
|
+
add(reasons, "ran_at_load", load_only, "code ran only at load; no mutant was judged")
|
|
22
|
+
add(reasons, "unreachable", report.counts[:unreachable], "tests did not call the replacement definition")
|
|
14
23
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
15
24
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
16
25
|
add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
|
|
26
|
+
add(reasons, "fail_fast", report.early_stop&.unreached&.size.to_i,
|
|
27
|
+
"selected subjects did not run because --fail-fast stopped the run")
|
|
28
|
+
inline_scope(reasons, report)
|
|
17
29
|
reasons
|
|
18
30
|
end
|
|
19
31
|
|
|
20
32
|
private
|
|
21
33
|
|
|
34
|
+
def inline_scope(reasons, report)
|
|
35
|
+
add(reasons, "inline_suppressed", report.inline_suppressions.size,
|
|
36
|
+
"candidates were suppressed by inline directives; see the full report audit")
|
|
37
|
+
end
|
|
38
|
+
|
|
22
39
|
def add(reasons, code, count, meaning)
|
|
23
40
|
reasons << { reason: code, count: count, meaning: meaning } if count.positive?
|
|
24
41
|
end
|
|
25
42
|
|
|
26
43
|
def skipped_for_coverage(report)
|
|
27
|
-
report.skipped_subjects.
|
|
44
|
+
report.skipped_subjects.values.count(NO_COVERAGE) + report.counts[:skipped]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def unsupported_subjects(report)
|
|
48
|
+
report.skipped_subjects.values.count do |reason|
|
|
49
|
+
reason != NO_COVERAGE && reason != Selection::CoverageMap::RAN_AT_LOAD
|
|
50
|
+
end
|
|
28
51
|
end
|
|
29
52
|
|
|
30
53
|
def omitted_by_cap(report)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "findings"
|
|
5
|
+
require_relative "survivor_hints"
|
|
5
6
|
require_relative "agent_incomplete"
|
|
6
7
|
require_relative "agent_clipping"
|
|
7
8
|
|
|
@@ -26,7 +27,7 @@ module Saboteur
|
|
|
26
27
|
scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
|
|
27
28
|
incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
|
|
28
29
|
findings: findings(report, shown, snippet_limit),
|
|
29
|
-
warnings: warnings(
|
|
30
|
+
warnings: report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] },
|
|
30
31
|
next: next_steps(report, provenance)
|
|
31
32
|
}
|
|
32
33
|
end
|
|
@@ -34,7 +35,7 @@ module Saboteur
|
|
|
34
35
|
private
|
|
35
36
|
|
|
36
37
|
def status_of(report, incomplete)
|
|
37
|
-
return "not_applicable" if report.results.empty? &&
|
|
38
|
+
return "not_applicable" if report.results.empty? && incomplete.empty?
|
|
38
39
|
|
|
39
40
|
incomplete.empty? ? "complete" : "partial"
|
|
40
41
|
end
|
|
@@ -87,7 +88,8 @@ module Saboteur
|
|
|
87
88
|
{
|
|
88
89
|
survivors: live, shown: shown_findings(all[:survivors], shown, snippet_limit),
|
|
89
90
|
omitted: live - [shown, all[:survivors].size].min,
|
|
90
|
-
accepted: report.survivors.size - live, ordering: all[:ordering]
|
|
91
|
+
accepted: report.survivors.size - live, ordering: all[:ordering],
|
|
92
|
+
hintCaveat: SurvivorHints::CAVEAT
|
|
91
93
|
}
|
|
92
94
|
end
|
|
93
95
|
|
|
@@ -97,11 +99,15 @@ module Saboteur
|
|
|
97
99
|
.merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
|
|
98
100
|
original: clip(finding[:original], snippet_limit),
|
|
99
101
|
replacement: clip(finding[:replacement], snippet_limit))
|
|
102
|
+
.merge(finding_hints(finding))
|
|
100
103
|
end
|
|
101
104
|
end
|
|
102
105
|
|
|
103
|
-
def
|
|
104
|
-
|
|
106
|
+
def finding_hints(finding)
|
|
107
|
+
structured = SurvivorHints.structured(operator: finding[:operator], original: finding[:original],
|
|
108
|
+
replacement: finding[:replacement], coarse: finding[:coarse])
|
|
109
|
+
{ hint: SurvivorHints.for(finding[:operator], coarse: finding[:coarse]) }.compact
|
|
110
|
+
.merge(testHint: structured)
|
|
105
111
|
end
|
|
106
112
|
|
|
107
113
|
def next_steps(report, provenance)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "survivor_ranking"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
module Reporter
|
|
6
8
|
# Findings for a code-review surface such as the saboteur GitHub Action:
|
|
@@ -15,12 +17,7 @@ module Saboteur
|
|
|
15
17
|
# GitHub shows at most 10 annotations of one kind per step.
|
|
16
18
|
LIMIT = 10
|
|
17
19
|
|
|
18
|
-
|
|
19
|
-
# replaced the whole method body, which hints that no test pins the
|
|
20
|
-
# method's behavior - though it can also be a re-evaluation limit or an
|
|
21
|
-
# equivalent mutant, so this is a heuristic, not a guarantee. Ties break
|
|
22
|
-
# by file, line and digest, so one report always reads the same way.
|
|
23
|
-
ORDERING = "coarse survivors first, then file, line and digest"
|
|
20
|
+
ORDERING = SurvivorRanking::ORDERING
|
|
24
21
|
|
|
25
22
|
class << self
|
|
26
23
|
def build(report)
|
|
@@ -46,20 +43,8 @@ module Saboteur
|
|
|
46
43
|
[results.size - LIMIT, 0].max
|
|
47
44
|
end
|
|
48
45
|
|
|
49
|
-
# partition keeps the coarse group ahead; inside a group the key is
|
|
50
|
-
# total (no two mutants share a digest), so the order is stable.
|
|
51
46
|
def ordered(report, results)
|
|
52
|
-
|
|
53
|
-
.map { |group| group.sort_by { |result| tie_break(report, result) } }
|
|
54
|
-
.flatten
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def tie_break(report, result)
|
|
58
|
-
[path(report, result), first_line(result), result.digest]
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
def first_line(result)
|
|
62
|
-
result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
|
|
47
|
+
SurvivorRanking.ordered(report, results)
|
|
63
48
|
end
|
|
64
49
|
|
|
65
50
|
def finding(report, result)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# Refuses linked or non-regular history/report destinations before writes.
|
|
7
|
+
module HistoryPaths
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def validate(paths)
|
|
11
|
+
paths = paths.compact.map { |path| File.expand_path(path) }
|
|
12
|
+
identities = paths.map do |path|
|
|
13
|
+
reject_links(path)
|
|
14
|
+
next unless File.exist?(path)
|
|
15
|
+
|
|
16
|
+
stat = File.stat(path)
|
|
17
|
+
raise Error, "score history/report destination must be a regular file" unless stat.file?
|
|
18
|
+
|
|
19
|
+
[stat.dev, stat.ino]
|
|
20
|
+
end.compact
|
|
21
|
+
raise Error, "score history/report destinations alias the same file" unless identities.uniq == identities
|
|
22
|
+
rescue SystemCallError => e
|
|
23
|
+
raise Error, "score history/report paths cannot be verified: #{e.message}"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def reject_links(path)
|
|
27
|
+
loop do
|
|
28
|
+
raise Error, "score history/report paths must not contain symlinks" if File.symlink?(path)
|
|
29
|
+
|
|
30
|
+
parent = File.dirname(path)
|
|
31
|
+
break if parent == path
|
|
32
|
+
|
|
33
|
+
path = parent
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|