saboteur 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +772 -0
- data/README.md +221 -1072
- data/lib/saboteur/attribute_finder.rb +113 -0
- data/lib/saboteur/cli/exit_status.rb +32 -1
- data/lib/saboteur/cli/help_options.rb +2 -1
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +47 -3
- data/lib/saboteur/cli/run_budget_options.rb +3 -1
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +35 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +74 -55
- data/lib/saboteur/config.rb +102 -6
- data/lib/saboteur/config_file.rb +19 -7
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/custom_map.rb +12 -1
- data/lib/saboteur/dirty_tree_warning.rb +130 -0
- data/lib/saboteur/doctor.rb +13 -0
- data/lib/saboteur/engine/baseline_guard.rb +135 -21
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +20 -0
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_store.rb +58 -14
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +19 -1
- data/lib/saboteur/engine/distribution.rb +10 -0
- data/lib/saboteur/engine/early_stop_recording.rb +3 -2
- data/lib/saboteur/engine/gated_run.rb +37 -0
- data/lib/saboteur/engine/load_coverage.rb +63 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +115 -0
- data/lib/saboteur/engine/result_cache.rb +65 -8
- data/lib/saboteur/engine/run_budget.rb +32 -2
- data/lib/saboteur/engine/sandbox_policy.rb +59 -3
- data/lib/saboteur/engine/scheduler.rb +131 -17
- data/lib/saboteur/engine/selection_announcement.rb +59 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +380 -21
- data/lib/saboteur/engine/test_guess.rb +407 -27
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine.rb +103 -129
- data/lib/saboteur/execution/external_runner.rb +489 -0
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/marker_sweep.rb +80 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +19 -3
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/reevaluator.rb +75 -12
- data/lib/saboteur/execution/runner.rb +46 -7
- data/lib/saboteur/execution/sandbox.rb +25 -6
- data/lib/saboteur/execution/seatbelt.rb +233 -0
- data/lib/saboteur/execution/supervisor.rb +29 -11
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +23 -1
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +79 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +60 -9
- data/lib/saboteur/ignore_list.rb +29 -6
- data/lib/saboteur/integration/rspec.rb +223 -5
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +28 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +189 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +148 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +4 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +5 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +125 -12
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +10 -3
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/stryker.rb +27 -3
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +56 -8
- data/lib/saboteur/result.rb +19 -3
- data/lib/saboteur/selection/coverage_map.rb +120 -10
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +93 -11
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/subject.rb +42 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +14 -0
- metadata +39 -16
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "operator_visit"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module MutationSource
|
|
10
|
+
# Weakens a regex literal: drops one anchor (\A, \z, \Z, ^, $) at a time
|
|
11
|
+
# and swaps each + or * quantifier for the other (#135). Escaped
|
|
12
|
+
# characters, character classes, and lazy or possessive stacks stay
|
|
13
|
+
# untouched, and interpolated regexes stay out: their bytes are not a
|
|
14
|
+
# stable 1:1 map. Anchors make validations decisive, so a dropped anchor
|
|
15
|
+
# or a widened quantifier is exactly the mutant a weak test misses.
|
|
16
|
+
class RegexReplacement < Base
|
|
17
|
+
include OperatorVisit
|
|
18
|
+
|
|
19
|
+
# Base inherits Object's empty initializer; there is no parent state to set.
|
|
20
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
21
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def name
|
|
25
|
+
"regex_replacement"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
BACKSLASH = 0x5C
|
|
29
|
+
ANCHOR_ESCAPES = %w[\\A \\z \\Z].freeze
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def mutate_node(node, subject, found)
|
|
34
|
+
return unless node.is_a?(Prism::RegularExpressionNode)
|
|
35
|
+
|
|
36
|
+
base = node.content_loc.start_offset
|
|
37
|
+
scan_anchors_and_quantifiers(node.content) do |offset, length, replacement|
|
|
38
|
+
edit = Edit.new(file: subject.file, start_byte: base + offset, end_byte: base + offset + length,
|
|
39
|
+
replacement: replacement)
|
|
40
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Yields [offset, length, replacement] for each anchor drop and
|
|
45
|
+
# quantifier swap, scanning bytes so offsets line up with the file.
|
|
46
|
+
def scan_anchors_and_quantifiers(pattern)
|
|
47
|
+
bytes = pattern.bytes
|
|
48
|
+
depth = 0
|
|
49
|
+
index = 0
|
|
50
|
+
while index < bytes.size
|
|
51
|
+
if bytes[index] == BACKSLASH
|
|
52
|
+
yield index, 2, "" if depth.zero? && ANCHOR_ESCAPES.include?(pattern.byteslice(index, 2))
|
|
53
|
+
index += 2
|
|
54
|
+
else
|
|
55
|
+
depth = scan_plain_byte(bytes, index, depth) { |offset, replacement| yield offset, 1, replacement }
|
|
56
|
+
index += 1
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# One non-escape byte; returns the character-class depth after it.
|
|
62
|
+
def scan_plain_byte(bytes, index, depth)
|
|
63
|
+
case bytes[index]
|
|
64
|
+
# [ opens a class only at top level or as a POSIX [:class:];
|
|
65
|
+
# inside a class it is a literal member.
|
|
66
|
+
when 0x5B then depth.zero? || bytes[index + 1] == 0x3A ? depth + 1 : depth
|
|
67
|
+
when 0x5D then depth.positive? ? depth - 1 : 0 # ]
|
|
68
|
+
when 0x5E, 0x24 # ^ $
|
|
69
|
+
yield index, "" if depth.zero?
|
|
70
|
+
depth
|
|
71
|
+
when 0x2B, 0x2A # + *
|
|
72
|
+
yield index, (bytes[index] == 0x2B ? "*" : "+") if depth.zero? && plain_quantifier?(bytes, index)
|
|
73
|
+
depth
|
|
74
|
+
else
|
|
75
|
+
depth
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# A bare quantifier: not the start of the pattern, not after an open
|
|
80
|
+
# group or alternation, and not stacked into a lazy (+?, ??) or
|
|
81
|
+
# possessive (*+, ++) form on either side.
|
|
82
|
+
def plain_quantifier?(bytes, index)
|
|
83
|
+
return false if index.zero?
|
|
84
|
+
|
|
85
|
+
previous = bytes[index - 1]
|
|
86
|
+
return false if [0x28, 0x7C, 0x3F, 0x2B, 0x2A].include?(previous) # ( | ? + *
|
|
87
|
+
|
|
88
|
+
nxt = bytes[index + 1]
|
|
89
|
+
nxt != 0x2B && nxt != 0x2A && nxt != 0x3F # + * ?
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/saboteur/preview.rb
CHANGED
|
@@ -1,30 +1,40 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "json"
|
|
4
5
|
require_relative "engine"
|
|
5
6
|
require_relative "text"
|
|
7
|
+
require_relative "version"
|
|
6
8
|
|
|
7
9
|
module Saboteur
|
|
8
10
|
# Source-only planning. Never boots the host or executes its test suite.
|
|
9
11
|
class Preview
|
|
10
|
-
|
|
12
|
+
# Version of the --format json inventory (#469). Consumers ignore fields
|
|
13
|
+
# they do not know; only a breaking change raises this number.
|
|
14
|
+
SCHEMA_VERSION = 1
|
|
15
|
+
|
|
16
|
+
def initialize(config, out:, err:, format: "text")
|
|
11
17
|
@config = config
|
|
12
18
|
@out = out
|
|
13
19
|
@diagnostics = Engine::Diagnostics.new(err)
|
|
20
|
+
@json = format == "json"
|
|
21
|
+
@inventory = []
|
|
22
|
+
@skipped = []
|
|
14
23
|
end
|
|
15
24
|
|
|
16
25
|
def call
|
|
17
26
|
announce_operators
|
|
18
27
|
selection = Engine::SubjectSelection.new(@config, @diagnostics)
|
|
19
28
|
subjects = selection.select(@config.included_files)
|
|
20
|
-
scope = selection.
|
|
29
|
+
scope = selection.behavior_lines && Engine::DiffScope.new(selection.behavior_lines)
|
|
21
30
|
guard = MutationSource::HazardGuard.new(@config["hazard"]) unless @config["allow_hazardous"]
|
|
22
31
|
candidates = Engine::Candidates.new(Engine::Candidates.build_sources(@config), @diagnostics, hazard_guard: guard)
|
|
23
|
-
lists = subjects.map { |subject| planned(subject, candidates, scope) }
|
|
32
|
+
lists = shard_lists(subjects, subjects.map { |subject| planned(subject, candidates, scope) })
|
|
24
33
|
inline_suppressions
|
|
25
34
|
incomplete = @diagnostics.discovery_failures.size
|
|
26
|
-
|
|
27
|
-
|
|
35
|
+
return print_inventory(subjects, lists, incomplete) if @json
|
|
36
|
+
|
|
37
|
+
print_totals(lists, incomplete)
|
|
28
38
|
incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
|
|
29
39
|
print_warnings
|
|
30
40
|
nil
|
|
@@ -32,25 +42,37 @@ module Saboteur
|
|
|
32
42
|
|
|
33
43
|
private
|
|
34
44
|
|
|
45
|
+
def shard_lists(subjects, lists)
|
|
46
|
+
return lists unless @config["shard"]
|
|
47
|
+
|
|
48
|
+
@shard = Engine::Shard.parse(@config["shard"])
|
|
49
|
+
@shard.select(subjects, lists)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def print_totals(lists, incomplete)
|
|
53
|
+
@out.puts(@shard.line) if @shard
|
|
54
|
+
@out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
|
|
55
|
+
"dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
|
|
56
|
+
end
|
|
57
|
+
|
|
35
58
|
def print_warnings
|
|
36
59
|
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
37
60
|
end
|
|
38
61
|
|
|
39
62
|
def announce_operators
|
|
40
|
-
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
|
|
63
|
+
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}") unless @json
|
|
41
64
|
end
|
|
42
65
|
|
|
43
66
|
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
44
67
|
# Name that scope rather than implying these rows were executed or scored.
|
|
45
68
|
def inline_suppressions
|
|
46
69
|
entries = @diagnostics.inline_suppressions
|
|
47
|
-
return if entries.empty?
|
|
70
|
+
return if entries.empty? || @json
|
|
48
71
|
|
|
49
72
|
@out.puts("Inline suppressions: #{entries.size} candidates in selected methods " \
|
|
50
73
|
"(before diff restriction and per-method cap; outside the planned total)")
|
|
51
74
|
entries.each do |entry|
|
|
52
|
-
|
|
53
|
-
@out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
75
|
+
@out.puts(" #{relative_path(entry[:file])}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
54
76
|
"#{entry[:digest]}: #{entry[:reason]}")
|
|
55
77
|
end
|
|
56
78
|
end
|
|
@@ -103,14 +125,105 @@ module Saboteur
|
|
|
103
125
|
end
|
|
104
126
|
|
|
105
127
|
def planned(subject, candidates, scope)
|
|
128
|
+
if subject.skip_reason
|
|
129
|
+
@diagnostics.warn("#{subject.id}: skipped (#{subject.skip_reason}); excluded from the planned total")
|
|
130
|
+
return []
|
|
131
|
+
end
|
|
132
|
+
|
|
106
133
|
list = candidates.for(subject)
|
|
107
134
|
list = scope.restrict(subject, list) if scope
|
|
108
135
|
limit = @config["max_mutants_per_method"]
|
|
109
|
-
|
|
136
|
+
if list.size > limit
|
|
137
|
+
@diagnostics.record_omission(subject, count: list.size - limit, limit: limit)
|
|
138
|
+
list.drop(limit).each { |(mutation, _)| record_skip(subject, mutation, "max_mutants_per_method") } if @json
|
|
139
|
+
end
|
|
110
140
|
list = list.first(limit)
|
|
111
|
-
|
|
112
|
-
|
|
141
|
+
if @json
|
|
142
|
+
list.each { |(mutation, _)| @inventory << mutant_entry(subject, mutation) }
|
|
143
|
+
else
|
|
144
|
+
@out.puts("#{relative_path(subject.file)}:#{subject.start_line} #{subject.id} #{list.size} mutants")
|
|
145
|
+
end
|
|
113
146
|
list
|
|
114
147
|
end
|
|
148
|
+
|
|
149
|
+
# The machine-readable inventory (#469): every planned mutant named by
|
|
150
|
+
# the stable digest a run or --id replay computes, plus the candidates
|
|
151
|
+
# that were planned but not runnable, each with its skip reason.
|
|
152
|
+
def print_inventory(subjects, lists, incomplete)
|
|
153
|
+
@out.puts(JSON.pretty_generate(
|
|
154
|
+
schema_version: SCHEMA_VERSION,
|
|
155
|
+
saboteur_version: VERSION,
|
|
156
|
+
mode: "dry-run",
|
|
157
|
+
operators: @config.operator_policy["effective"],
|
|
158
|
+
test_selection: @config["test_selection"],
|
|
159
|
+
tests: @config["tests"],
|
|
160
|
+
subjects: subjects.zip(lists).map { |subject, list| subject_entry(subject, list) },
|
|
161
|
+
mutants: @inventory,
|
|
162
|
+
skipped: @skipped + suppression_entries,
|
|
163
|
+
totals: {
|
|
164
|
+
mutants: lists.sum(&:size),
|
|
165
|
+
dropped_by_max_mutants_per_method: @diagnostics.omissions.sum(&:count),
|
|
166
|
+
incomplete_files: incomplete
|
|
167
|
+
},
|
|
168
|
+
estimated_seconds: incomplete.zero? ? estimated_seconds(subjects, lists) : nil,
|
|
169
|
+
warnings: @diagnostics.warnings
|
|
170
|
+
))
|
|
171
|
+
nil
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def estimated_seconds(subjects, lists)
|
|
175
|
+
map = cached_map
|
|
176
|
+
seconds = map && work_seconds(map, subjects, lists)
|
|
177
|
+
seconds&.round(2)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def subject_entry(subject, list)
|
|
181
|
+
{ id: subject.id, file: relative_path(subject.file), start_line: subject.start_line,
|
|
182
|
+
end_line: subject.end_line, mutants: list.size }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Byte offsets index the file as read from disk, like the Stryker
|
|
186
|
+
# report; original and replacement text make each edit self-contained.
|
|
187
|
+
def mutant_entry(subject, mutation)
|
|
188
|
+
{
|
|
189
|
+
digest: mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source),
|
|
190
|
+
subject: subject.id,
|
|
191
|
+
operator: mutation.operator.to_s,
|
|
192
|
+
file: relative_path(subject.file),
|
|
193
|
+
line: subject.line_of(mutation.edits.first.start_byte),
|
|
194
|
+
edits: mutation.edits.map do |edit|
|
|
195
|
+
{ start_byte: edit.start_byte, end_byte: edit.end_byte,
|
|
196
|
+
original: source_span(subject, edit), replacement: edit.replacement }
|
|
197
|
+
end
|
|
198
|
+
}
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def record_skip(subject, mutation, reason)
|
|
202
|
+
@skipped << mutant_entry(subject, mutation).merge(reason: reason)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Inline-suppressed and hazard-refused candidates with their reasons,
|
|
206
|
+
# recorded during candidate filtering, before diff restriction and the
|
|
207
|
+
# per-method cap (the same scope the text audit names).
|
|
208
|
+
def suppression_entries
|
|
209
|
+
inline = @diagnostics.inline_suppressions.map do |entry|
|
|
210
|
+
{ digest: entry[:digest], subject: entry[:subject], operator: entry[:operator].to_s,
|
|
211
|
+
file: relative_path(entry[:file]), line: entry[:line], reason: "inline_suppression: #{entry[:reason]}" }
|
|
212
|
+
end
|
|
213
|
+
hazards = @diagnostics.generation_events.select { |event| event.source == "hazard_guard" }.map do |event|
|
|
214
|
+
{ subject: event.subject, operator: event.operator.to_s, file: relative_path(event.file),
|
|
215
|
+
reason: "hazardous" }
|
|
216
|
+
end
|
|
217
|
+
inline + hazards
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def relative_path(file)
|
|
221
|
+
file.delete_prefix("#{@config.root}/")
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def source_span(subject, edit)
|
|
225
|
+
subject.file_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte).to_s
|
|
226
|
+
.force_encoding(Encoding::UTF_8)
|
|
227
|
+
end
|
|
115
228
|
end
|
|
116
229
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# The lexical `using` sites of one parsed file (#634). A bare `using`
|
|
6
|
+
# activates its refinement from where it sits to the end of its own
|
|
7
|
+
# scope: a class or module body, or the whole file at the top level.
|
|
8
|
+
# Activation is lexical, not namespaced: a `class ::Q` nested inside
|
|
9
|
+
# the body still sees it, while a reopened body with the same constant
|
|
10
|
+
# path after the scope's `end` does not. Sites therefore compare byte
|
|
11
|
+
# ranges, not constant paths: a site covers a subject that starts after
|
|
12
|
+
# the `using` and before the scope's closing `end`. A `using` inside a
|
|
13
|
+
# method or block body cannot be scoped this way, so one anywhere keeps
|
|
14
|
+
# the old whole-file skip. Skipping only ever means "not mutated",
|
|
15
|
+
# never a wrong verdict.
|
|
16
|
+
class RefinementActivations
|
|
17
|
+
Site = Data.define(:start_byte, :scope_end, :dynamic_scope) do
|
|
18
|
+
def covers?(subject)
|
|
19
|
+
!dynamic_scope && subject.start_byte > start_byte && subject.start_byte < scope_end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.mark(subjects, program)
|
|
24
|
+
sites = []
|
|
25
|
+
collect(program, program.end_offset, false, sites)
|
|
26
|
+
return if sites.empty?
|
|
27
|
+
|
|
28
|
+
if sites.any?(&:dynamic_scope)
|
|
29
|
+
subjects.each { |subject| subject.skip_reason = "refinements" }
|
|
30
|
+
return
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
subjects.each do |subject|
|
|
34
|
+
subject.skip_reason = "refinements" if sites.any? { |site| site.covers?(subject) }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.collect(node, scope_end, dynamic, sites)
|
|
39
|
+
if node.is_a?(Prism::CallNode) && node.name == :using && node.receiver.nil?
|
|
40
|
+
sites << Site.new(start_byte: node.location.start_offset, scope_end: scope_end, dynamic_scope: dynamic)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
descend(node, scope_end, dynamic, sites)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def self.descend(node, scope_end, dynamic, sites)
|
|
47
|
+
case node
|
|
48
|
+
when Prism::ClassNode
|
|
49
|
+
collect(node.superclass, scope_end, dynamic, sites) if node.superclass
|
|
50
|
+
collect(node.body, node.end_offset, dynamic, sites) if node.body
|
|
51
|
+
when Prism::ModuleNode, Prism::SingletonClassNode
|
|
52
|
+
collect(node.body, node.end_offset, dynamic, sites) if node.body
|
|
53
|
+
when Prism::DefNode, Prism::BlockNode, Prism::LambdaNode
|
|
54
|
+
node.compact_child_nodes.each { |child| collect(child, scope_end, true, sites) }
|
|
55
|
+
else
|
|
56
|
+
node.compact_child_nodes.each { |child| collect(child, scope_end, dynamic, sites) }
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -32,8 +32,16 @@ module Saboteur
|
|
|
32
32
|
|
|
33
33
|
# Where a verdict comes from: the exact selection and the state of the working
|
|
34
34
|
# source before the run started. subjects, tests and sources are arrays.
|
|
35
|
+
# typed_settings: the settings keys typed on the command line, so the
|
|
36
|
+
# generated commands can tell a typed empty list (which may clear a
|
|
37
|
+
# config-file value) from one the config itself supplies.
|
|
35
38
|
Provenance = Data.define(:ref, :seed, :head, :working_tree, :changed_files, :mode, :json_report, :subjects,
|
|
36
|
-
:tests, :sources, :config_path, :digest, :cache, :incremental, :settings
|
|
39
|
+
:tests, :sources, :config_path, :digest, :cache, :incremental, :settings,
|
|
40
|
+
:typed_settings) do
|
|
41
|
+
def initialize(typed_settings: [], **fields)
|
|
42
|
+
super
|
|
43
|
+
end
|
|
44
|
+
end
|
|
37
45
|
|
|
38
46
|
class << self
|
|
39
47
|
def call(report, provenance)
|
|
@@ -19,7 +19,8 @@ module Saboteur
|
|
|
19
19
|
SETTING_FLAGS = { reeval: "--reeval", integration: "--integration", jobs: "--jobs",
|
|
20
20
|
timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html",
|
|
21
21
|
operators: "--operators", exclude_operators: "--exclude-operators",
|
|
22
|
-
baseline_passes: "--baseline-passes", max_duration: "--max-duration"
|
|
22
|
+
baseline_passes: "--baseline-passes", max_duration: "--max-duration",
|
|
23
|
+
shard: "--shard" }.freeze
|
|
23
24
|
|
|
24
25
|
# CLI-only boolean switches recorded in settings, mapped by value, so
|
|
25
26
|
# an explicit false (--no-max-duration-ok over a config true) survives
|
|
@@ -70,11 +71,17 @@ module Saboteur
|
|
|
70
71
|
tests.flat_map { |test| ["--test", test] }
|
|
71
72
|
end
|
|
72
73
|
|
|
73
|
-
|
|
74
|
+
# An empty list nobody typed came from the config file or its
|
|
75
|
+
# default, which the command reads again, so its flag is left out
|
|
76
|
+
# (#317). A typed empty list stays: it may clear a config value.
|
|
77
|
+
def setting_words(provenance, replay: false)
|
|
74
78
|
SETTING_FLAGS.flat_map do |key, flag|
|
|
79
|
+
next [] if replay && key == :shard
|
|
75
80
|
next [] unless provenance.settings.key?(key)
|
|
76
81
|
|
|
77
82
|
value = provenance.settings[key]
|
|
83
|
+
next [] if value == [] && !provenance.typed_settings.include?(key)
|
|
84
|
+
|
|
78
85
|
[flag, value.is_a?(Array) ? value.join(",") : value.to_s]
|
|
79
86
|
end
|
|
80
87
|
end
|
|
@@ -88,7 +95,7 @@ module Saboteur
|
|
|
88
95
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
89
96
|
words << "--all" if provenance.settings[:all] && provenance.tests.empty?
|
|
90
97
|
words.push(*test_words(provenance.tests))
|
|
91
|
-
words.concat(setting_words(provenance))
|
|
98
|
+
words.concat(setting_words(provenance, replay: replay))
|
|
92
99
|
words.concat(mode_words(provenance))
|
|
93
100
|
words.push("--id", provenance.digest) if provenance.digest && !replay
|
|
94
101
|
words
|
|
@@ -14,13 +14,17 @@ module Saboteur
|
|
|
14
14
|
class << self
|
|
15
15
|
def reasons(report)
|
|
16
16
|
reasons = []
|
|
17
|
+
shard_reason(reasons, report)
|
|
17
18
|
add(reasons, "no_coverage", skipped_for_coverage(report), "subjects or mutants had no covering test")
|
|
18
|
-
load_only = report.counts[:ran_at_load] + report.skipped_subjects.values.count do |reason|
|
|
19
|
-
reason == Selection::CoverageMap::RAN_AT_LOAD
|
|
20
|
-
end
|
|
21
19
|
add(reasons, "unsupported", unsupported_subjects(report),
|
|
22
20
|
"subjects were skipped as unsupported (for example defined inside a block); no mutant was judged")
|
|
23
|
-
add(reasons, "ran_at_load",
|
|
21
|
+
add(reasons, "ran_at_load", load_only_subjects(report), "code ran only at load; no mutant was judged")
|
|
22
|
+
# A file loaded before Coverage.start has no attribution at all:
|
|
23
|
+
# not unsupported (the shape is fine) and not ran_at_load (the
|
|
24
|
+
# load-only evidence does not exist) - its own code (#688).
|
|
25
|
+
add(reasons, "preloaded", preloaded_subjects(report),
|
|
26
|
+
"files loaded before coverage started; their subjects could not be attributed to tests; " \
|
|
27
|
+
"no mutant was judged")
|
|
24
28
|
add(reasons, "unreachable", report.counts[:unreachable], "tests did not call the replacement definition")
|
|
25
29
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
26
30
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
@@ -35,6 +39,16 @@ module Saboteur
|
|
|
35
39
|
|
|
36
40
|
private
|
|
37
41
|
|
|
42
|
+
# A shard always runs a subset of the planned inventory by contract, even
|
|
43
|
+
# when this shard happened to select every planned mutant (count 0).
|
|
44
|
+
def shard_reason(reasons, report)
|
|
45
|
+
shard = report.shard
|
|
46
|
+
return unless shard
|
|
47
|
+
|
|
48
|
+
reasons << { reason: "shard", count: shard.planned_count - shard.selected_count,
|
|
49
|
+
meaning: "planned mutants assigned to other shards" }
|
|
50
|
+
end
|
|
51
|
+
|
|
38
52
|
# Source the selection could not read at all: its subjects never
|
|
39
53
|
# entered the run, so a clean or complete verdict would be wrong.
|
|
40
54
|
def discovery_failures(reasons, report)
|
|
@@ -69,9 +83,21 @@ module Saboteur
|
|
|
69
83
|
report.skipped_subjects.values.count(NO_COVERAGE) + report.counts[:skipped]
|
|
70
84
|
end
|
|
71
85
|
|
|
86
|
+
# Load-only mutants plus the subjects skipped for the same reason.
|
|
87
|
+
def load_only_subjects(report)
|
|
88
|
+
report.counts[:ran_at_load] + report.skipped_subjects.values.count do |reason|
|
|
89
|
+
reason == Selection::CoverageMap::RAN_AT_LOAD
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def preloaded_subjects(report)
|
|
94
|
+
report.skipped_subjects.values.count(Selection::CoverageMap::PRELOADED)
|
|
95
|
+
end
|
|
96
|
+
|
|
72
97
|
def unsupported_subjects(report)
|
|
73
98
|
report.skipped_subjects.values.count do |reason|
|
|
74
|
-
reason != NO_COVERAGE && reason != Selection::CoverageMap::RAN_AT_LOAD
|
|
99
|
+
reason != NO_COVERAGE && reason != Selection::CoverageMap::RAN_AT_LOAD &&
|
|
100
|
+
reason != Selection::CoverageMap::PRELOADED
|
|
75
101
|
end
|
|
76
102
|
end
|
|
77
103
|
|
|
@@ -26,10 +26,11 @@ module Saboteur
|
|
|
26
26
|
counts: counts(report), score: score(report),
|
|
27
27
|
scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
|
|
28
28
|
incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
|
|
29
|
-
findings: findings(report, shown, snippet_limit)
|
|
29
|
+
findings: findings(report, shown, snippet_limit)
|
|
30
|
+
}.merge(report.shard ? { shard: report.shard.to_h } : {}).merge(
|
|
30
31
|
warnings: report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] },
|
|
31
32
|
next: next_steps(report, provenance)
|
|
32
|
-
|
|
33
|
+
)
|
|
33
34
|
end
|
|
34
35
|
|
|
35
36
|
private
|
|
@@ -95,7 +96,7 @@ module Saboteur
|
|
|
95
96
|
|
|
96
97
|
def shown_findings(listed, shown, snippet_limit)
|
|
97
98
|
listed.first(shown).map do |finding|
|
|
98
|
-
finding.slice(:digest, :line, :endLine, :operator, :coarse)
|
|
99
|
+
finding.slice(:digest, :line, :endLine, :operator, :coarse, :classification)
|
|
99
100
|
.merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
|
|
100
101
|
original: clip(finding[:original], snippet_limit),
|
|
101
102
|
replacement: clip(finding[:replacement], snippet_limit))
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "survivor_ranking"
|
|
5
|
+
require_relative "survivor_classification"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
module Reporter
|
|
@@ -59,8 +60,10 @@ module Saboteur
|
|
|
59
60
|
original: joined_texts(result) { |edit| original_text(result, edit) },
|
|
60
61
|
replacement: joined_texts(result, &:replacement),
|
|
61
62
|
edits: entries,
|
|
62
|
-
coarse: result.mutation.coarse
|
|
63
|
-
|
|
63
|
+
coarse: result.mutation.coarse,
|
|
64
|
+
# Why the survivor is worth a look, as a label only (#192).
|
|
65
|
+
classification: SurvivorClassification.label(result)
|
|
66
|
+
}.compact
|
|
64
67
|
entry = with_changed_lines(report, result, entry)
|
|
65
68
|
result.ignored ? entry.merge(reason: result.ignored) : entry
|
|
66
69
|
end
|
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
module Saboteur
|
|
5
5
|
module Reporter
|
|
6
6
|
# v1 stays readable, but v2 records require versioned operator evidence.
|
|
7
|
+
# A policy stamped with an older POLICY_VERSION is shape-checked only:
|
|
8
|
+
# the operator set has moved on, so it never recomputes equal - and it
|
|
9
|
+
# never compares equal to a current run's policy either, which is what
|
|
10
|
+
# keeps trend from mixing operator sets (#135).
|
|
7
11
|
module HistoryPolicy
|
|
8
12
|
def validate_header(record, fields)
|
|
9
13
|
version = record.is_a?(Hash) && record["version"]
|
|
@@ -18,6 +22,11 @@ module Saboteur
|
|
|
18
22
|
raise Error, "score history contains invalid operator policy" unless value.is_a?(Hash)
|
|
19
23
|
|
|
20
24
|
fields(value, %w[version selected excluded effective])
|
|
25
|
+
policy_version = value["version"]
|
|
26
|
+
unless policy_version.is_a?(Integer) &&
|
|
27
|
+
policy_version.between?(1, MutationSource::OperatorRegistry::POLICY_VERSION)
|
|
28
|
+
raise Error, "score history contains invalid operator policy"
|
|
29
|
+
end
|
|
21
30
|
raise Error, "score history contains invalid operator policy" if value["effective"].empty?
|
|
22
31
|
|
|
23
32
|
lists = value.values_at("selected", "excluded", "effective")
|
|
@@ -25,6 +34,8 @@ module Saboteur
|
|
|
25
34
|
raise Error, "score history contains invalid operator policy"
|
|
26
35
|
end
|
|
27
36
|
|
|
37
|
+
return if policy_version != MutationSource::OperatorRegistry::POLICY_VERSION
|
|
38
|
+
|
|
28
39
|
expected = MutationSource::OperatorRegistry.policy(value["selected"], value["excluded"])
|
|
29
40
|
raise Error, "score history contains invalid operator policy" unless value == expected
|
|
30
41
|
rescue ConfigError, NoMethodError, TypeError
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "score_history"
|
|
5
|
+
require_relative "history_record"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Reporter
|
|
9
|
+
# Read side of the append-only score history for `saboteur badge` and
|
|
10
|
+
# `saboteur trend` (#474). Every record is re-validated on read and the
|
|
11
|
+
# file discipline mirrors ScoreHistory: regular file, no symlink,
|
|
12
|
+
# byte-capped lines, no repair of imported evidence.
|
|
13
|
+
module HistoryReader
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def read(path)
|
|
17
|
+
records = []
|
|
18
|
+
File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |file|
|
|
19
|
+
raise Error, "score history must be a regular file" unless file.stat.file?
|
|
20
|
+
|
|
21
|
+
file.flock(File::LOCK_SH)
|
|
22
|
+
limit = ScoreHistory::MAX_RECORD_BYTES
|
|
23
|
+
while (line = file.gets(limit + 1))
|
|
24
|
+
raise Error, "score history record exceeds byte limit" if line.bytesize > limit
|
|
25
|
+
raise Error, "score history contains a truncated record" unless line.end_with?("\n")
|
|
26
|
+
|
|
27
|
+
records << HistoryRecord.parse(line)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
records
|
|
31
|
+
rescue SystemCallError, IOError => e
|
|
32
|
+
raise Error, "score history cannot be read: #{e.message}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The ScoreHistory change rules: same record version, selection,
|
|
36
|
+
# inventory and operator policy (a null policy never compares), both
|
|
37
|
+
# runs complete.
|
|
38
|
+
def comparable?(previous, current)
|
|
39
|
+
previous["version"] == current["version"] &&
|
|
40
|
+
previous["selection"] == current["selection"] &&
|
|
41
|
+
previous["inventory"] == current["inventory"] &&
|
|
42
|
+
previous["incomplete"].empty? && current["incomplete"].empty? &&
|
|
43
|
+
!previous["operatorPolicy"].nil? && previous["operatorPolicy"] == current["operatorPolicy"]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
require "cgi"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
|
|
7
|
+
require_relative "survivor_classification"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
module Reporter
|
|
9
11
|
# One self-contained HTML page: inline style, no script, no network. It is
|
|
@@ -23,7 +25,9 @@ module Saboteur
|
|
|
23
25
|
.status{border-radius:3px;padding:0 .4rem;color:#fff;white-space:nowrap}
|
|
24
26
|
.Killed{background:#2e7d32}.Survived{background:#c62828}.Timeout{background:#b45309}
|
|
25
27
|
.RuntimeError{background:#6a1b9a}.NoCoverage{background:#455a64}.Ignored{background:#616161}
|
|
26
|
-
.Accepted{background:#8d6e63}
|
|
28
|
+
.Accepted,.Triage_accepted_equivalent{background:#8d6e63}
|
|
29
|
+
.Triage_wont_fix{background:#616161}.Triage_todo{background:#b45309}
|
|
30
|
+
.classification{border-radius:3px;padding:0 .4rem;background:#eceff1;color:#37474f;white-space:nowrap}
|
|
27
31
|
.warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
|
|
28
32
|
code,pre{font-family:ui-monospace,monospace;font-size:13px}
|
|
29
33
|
pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
|
|
@@ -195,12 +199,30 @@ module Saboteur
|
|
|
195
199
|
def row_status(mutant, result)
|
|
196
200
|
badge = status_label(mutant[:status])
|
|
197
201
|
# Only survivors are ever put on the ignore list.
|
|
198
|
-
result.ignored
|
|
202
|
+
if result.ignored || result.triage_state
|
|
203
|
+
# An entry without a state predates triage states: render Accepted.
|
|
204
|
+
badge = if result.triage_state.nil?
|
|
205
|
+
"#{badge} #{status_label('Accepted')}"
|
|
206
|
+
else
|
|
207
|
+
"#{badge} #{status_label(triage_badge(result), css_class: "Triage_#{result.triage_state}")}"
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
# The survivor's classification, as a label only (#192).
|
|
211
|
+
label = SurvivorClassification.label(result)
|
|
212
|
+
label ? "#{badge} <span class=\"classification\">#{escape(label)}</span>" : badge
|
|
199
213
|
end
|
|
200
214
|
|
|
201
215
|
# One edit shows the original text and its replacement. A mutant with
|
|
202
216
|
# several edits shows only the replacements, because one location cannot
|
|
203
217
|
# stand for all of them.
|
|
218
|
+
def triage_badge(result)
|
|
219
|
+
case result.triage_state
|
|
220
|
+
when "wont_fix" then "Won\u2019t fix"
|
|
221
|
+
when "todo" then "Todo"
|
|
222
|
+
else "Accepted"
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
204
226
|
def change(mutant, result)
|
|
205
227
|
edits = result.mutation.edits
|
|
206
228
|
return "<pre>#{escape(mutant[:replacement])}</pre>" unless edits.size == 1
|
|
@@ -212,8 +234,8 @@ module Saboteur
|
|
|
212
234
|
"<pre>#{escape(edit.replacement)}</pre>"
|
|
213
235
|
end
|
|
214
236
|
|
|
215
|
-
def status_label(status)
|
|
216
|
-
%(<span class="status #{
|
|
237
|
+
def status_label(status, css_class: status)
|
|
238
|
+
%(<span class="status #{css_class}">#{status}</span>)
|
|
217
239
|
end
|
|
218
240
|
|
|
219
241
|
def escape(text)
|
|
@@ -20,6 +20,12 @@ module Saboteur
|
|
|
20
20
|
join(result, &:replacement)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# A mutant that only deletes: every edit's replacement is empty.
|
|
24
|
+
# Rendered as "removes X" rather than "X -> " with a blank side.
|
|
25
|
+
def removal?(result)
|
|
26
|
+
sorted_edits(result).all? { |edit| edit.replacement.empty? }
|
|
27
|
+
end
|
|
28
|
+
|
|
23
29
|
def sorted_edits(result)
|
|
24
30
|
result.mutation.edits.sort_by(&:start_byte)
|
|
25
31
|
end
|