saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "operator_visit"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module MutationSource
|
|
10
|
+
# Weakens a regex literal: drops one anchor (\A, \z, \Z, ^, $) at a time
|
|
11
|
+
# and swaps each + or * quantifier for the other (#135). Escaped
|
|
12
|
+
# characters, character classes, and lazy or possessive stacks stay
|
|
13
|
+
# untouched, and interpolated regexes stay out: their bytes are not a
|
|
14
|
+
# stable 1:1 map. Anchors make validations decisive, so a dropped anchor
|
|
15
|
+
# or a widened quantifier is exactly the mutant a weak test misses.
|
|
16
|
+
class RegexReplacement < Base
|
|
17
|
+
include OperatorVisit
|
|
18
|
+
|
|
19
|
+
# Base inherits Object's empty initializer; there is no parent state to set.
|
|
20
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
21
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def name
|
|
25
|
+
"regex_replacement"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
BACKSLASH = 0x5C
|
|
29
|
+
ANCHOR_ESCAPES = %w[\\A \\z \\Z].freeze
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def mutate_node(node, subject, found)
|
|
34
|
+
return unless node.is_a?(Prism::RegularExpressionNode)
|
|
35
|
+
|
|
36
|
+
base = node.content_loc.start_offset
|
|
37
|
+
scan_anchors_and_quantifiers(node.content) do |offset, length, replacement|
|
|
38
|
+
edit = Edit.new(file: subject.file, start_byte: base + offset, end_byte: base + offset + length,
|
|
39
|
+
replacement: replacement)
|
|
40
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Yields [offset, length, replacement] for each anchor drop and
|
|
45
|
+
# quantifier swap, scanning bytes so offsets line up with the file.
|
|
46
|
+
def scan_anchors_and_quantifiers(pattern)
|
|
47
|
+
bytes = pattern.bytes
|
|
48
|
+
depth = 0
|
|
49
|
+
index = 0
|
|
50
|
+
while index < bytes.size
|
|
51
|
+
if bytes[index] == BACKSLASH
|
|
52
|
+
yield index, 2, "" if depth.zero? && ANCHOR_ESCAPES.include?(pattern.byteslice(index, 2))
|
|
53
|
+
index += 2
|
|
54
|
+
else
|
|
55
|
+
depth = scan_plain_byte(bytes, index, depth) { |offset, replacement| yield offset, 1, replacement }
|
|
56
|
+
index += 1
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# One non-escape byte; returns the character-class depth after it.
|
|
62
|
+
def scan_plain_byte(bytes, index, depth)
|
|
63
|
+
case bytes[index]
|
|
64
|
+
# [ opens a class only at top level or as a POSIX [:class:];
|
|
65
|
+
# inside a class it is a literal member.
|
|
66
|
+
when 0x5B then depth.zero? || bytes[index + 1] == 0x3A ? depth + 1 : depth
|
|
67
|
+
when 0x5D then depth.positive? ? depth - 1 : 0 # ]
|
|
68
|
+
when 0x5E, 0x24 # ^ $
|
|
69
|
+
yield index, "" if depth.zero?
|
|
70
|
+
depth
|
|
71
|
+
when 0x2B, 0x2A # + *
|
|
72
|
+
yield index, (bytes[index] == 0x2B ? "*" : "+") if depth.zero? && plain_quantifier?(bytes, index)
|
|
73
|
+
depth
|
|
74
|
+
else
|
|
75
|
+
depth
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# A bare quantifier: not the start of the pattern, not after an open
|
|
80
|
+
# group or alternation, and not stacked into a lazy (+?, ??) or
|
|
81
|
+
# possessive (*+, ++) form on either side.
|
|
82
|
+
def plain_quantifier?(bytes, index)
|
|
83
|
+
return false if index.zero?
|
|
84
|
+
|
|
85
|
+
previous = bytes[index - 1]
|
|
86
|
+
return false if [0x28, 0x7C, 0x3F, 0x2B, 0x2A].include?(previous) # ( | ? + *
|
|
87
|
+
|
|
88
|
+
nxt = bytes[index + 1]
|
|
89
|
+
nxt != 0x2B && nxt != 0x2A && nxt != 0x3F # + * ?
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/saboteur/preview.rb
CHANGED
|
@@ -1,29 +1,40 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "json"
|
|
4
5
|
require_relative "engine"
|
|
6
|
+
require_relative "text"
|
|
7
|
+
require_relative "version"
|
|
5
8
|
|
|
6
9
|
module Saboteur
|
|
7
10
|
# Source-only planning. Never boots the host or executes its test suite.
|
|
8
11
|
class Preview
|
|
9
|
-
|
|
12
|
+
# Version of the --format json inventory (#469). Consumers ignore fields
|
|
13
|
+
# they do not know; only a breaking change raises this number.
|
|
14
|
+
SCHEMA_VERSION = 1
|
|
15
|
+
|
|
16
|
+
def initialize(config, out:, err:, format: "text")
|
|
10
17
|
@config = config
|
|
11
18
|
@out = out
|
|
12
19
|
@diagnostics = Engine::Diagnostics.new(err)
|
|
20
|
+
@json = format == "json"
|
|
21
|
+
@inventory = []
|
|
22
|
+
@skipped = []
|
|
13
23
|
end
|
|
14
24
|
|
|
15
25
|
def call
|
|
16
26
|
announce_operators
|
|
17
27
|
selection = Engine::SubjectSelection.new(@config, @diagnostics)
|
|
18
28
|
subjects = selection.select(@config.included_files)
|
|
19
|
-
scope = selection.
|
|
29
|
+
scope = selection.behavior_lines && Engine::DiffScope.new(selection.behavior_lines)
|
|
20
30
|
guard = MutationSource::HazardGuard.new(@config["hazard"]) unless @config["allow_hazardous"]
|
|
21
31
|
candidates = Engine::Candidates.new(Engine::Candidates.build_sources(@config), @diagnostics, hazard_guard: guard)
|
|
22
|
-
lists = subjects.map { |subject| planned(subject, candidates, scope) }
|
|
32
|
+
lists = shard_lists(subjects, subjects.map { |subject| planned(subject, candidates, scope) })
|
|
23
33
|
inline_suppressions
|
|
24
34
|
incomplete = @diagnostics.discovery_failures.size
|
|
25
|
-
|
|
26
|
-
|
|
35
|
+
return print_inventory(subjects, lists, incomplete) if @json
|
|
36
|
+
|
|
37
|
+
print_totals(lists, incomplete)
|
|
27
38
|
incomplete.zero? ? estimate(subjects, lists) : @out.puts("no estimate: source discovery was incomplete")
|
|
28
39
|
print_warnings
|
|
29
40
|
nil
|
|
@@ -31,25 +42,37 @@ module Saboteur
|
|
|
31
42
|
|
|
32
43
|
private
|
|
33
44
|
|
|
45
|
+
def shard_lists(subjects, lists)
|
|
46
|
+
return lists unless @config["shard"]
|
|
47
|
+
|
|
48
|
+
@shard = Engine::Shard.parse(@config["shard"])
|
|
49
|
+
@shard.select(subjects, lists)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def print_totals(lists, incomplete)
|
|
53
|
+
@out.puts(@shard.line) if @shard
|
|
54
|
+
@out.puts("Total: #{lists.sum(&:size)} mutants; #{@diagnostics.omissions.sum(&:count)} " \
|
|
55
|
+
"dropped by max_mutants_per_method#{incomplete_note(incomplete)}")
|
|
56
|
+
end
|
|
57
|
+
|
|
34
58
|
def print_warnings
|
|
35
59
|
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
36
60
|
end
|
|
37
61
|
|
|
38
62
|
def announce_operators
|
|
39
|
-
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
|
|
63
|
+
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}") unless @json
|
|
40
64
|
end
|
|
41
65
|
|
|
42
66
|
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
43
67
|
# Name that scope rather than implying these rows were executed or scored.
|
|
44
68
|
def inline_suppressions
|
|
45
69
|
entries = @diagnostics.inline_suppressions
|
|
46
|
-
return if entries.empty?
|
|
70
|
+
return if entries.empty? || @json
|
|
47
71
|
|
|
48
72
|
@out.puts("Inline suppressions: #{entries.size} candidates in selected methods " \
|
|
49
73
|
"(before diff restriction and per-method cap; outside the planned total)")
|
|
50
74
|
entries.each do |entry|
|
|
51
|
-
|
|
52
|
-
@out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
75
|
+
@out.puts(" #{relative_path(entry[:file])}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
53
76
|
"#{entry[:digest]}: #{entry[:reason]}")
|
|
54
77
|
end
|
|
55
78
|
end
|
|
@@ -92,7 +115,7 @@ module Saboteur
|
|
|
92
115
|
# coverage map means "no estimate", never an exception.
|
|
93
116
|
def cached_map
|
|
94
117
|
path = File.join(@config.path("cache_dir"), "preview.json")
|
|
95
|
-
receipt = JSON.parse(File.read(path))
|
|
118
|
+
receipt = JSON.parse(Text.utf8_strict(File.read(path)))
|
|
96
119
|
return unless receipt["fingerprint"] == Engine::RunFingerprint.new(config: @config).snapshot.digest
|
|
97
120
|
|
|
98
121
|
Selection::CoverageMap.load(receipt["path"])
|
|
@@ -105,11 +128,97 @@ module Saboteur
|
|
|
105
128
|
list = candidates.for(subject)
|
|
106
129
|
list = scope.restrict(subject, list) if scope
|
|
107
130
|
limit = @config["max_mutants_per_method"]
|
|
108
|
-
|
|
131
|
+
if list.size > limit
|
|
132
|
+
@diagnostics.record_omission(subject, count: list.size - limit, limit: limit)
|
|
133
|
+
list.drop(limit).each { |(mutation, _)| record_skip(subject, mutation, "max_mutants_per_method") } if @json
|
|
134
|
+
end
|
|
109
135
|
list = list.first(limit)
|
|
110
|
-
|
|
111
|
-
|
|
136
|
+
if @json
|
|
137
|
+
list.each { |(mutation, _)| @inventory << mutant_entry(subject, mutation) }
|
|
138
|
+
else
|
|
139
|
+
@out.puts("#{relative_path(subject.file)}:#{subject.start_line} #{subject.id} #{list.size} mutants")
|
|
140
|
+
end
|
|
112
141
|
list
|
|
113
142
|
end
|
|
143
|
+
|
|
144
|
+
# The machine-readable inventory (#469): every planned mutant named by
|
|
145
|
+
# the stable digest a run or --id replay computes, plus the candidates
|
|
146
|
+
# that were planned but not runnable, each with its skip reason.
|
|
147
|
+
def print_inventory(subjects, lists, incomplete)
|
|
148
|
+
@out.puts(JSON.pretty_generate(
|
|
149
|
+
schema_version: SCHEMA_VERSION,
|
|
150
|
+
saboteur_version: VERSION,
|
|
151
|
+
mode: "dry-run",
|
|
152
|
+
operators: @config.operator_policy["effective"],
|
|
153
|
+
test_selection: @config["test_selection"],
|
|
154
|
+
tests: @config["tests"],
|
|
155
|
+
subjects: subjects.zip(lists).map { |subject, list| subject_entry(subject, list) },
|
|
156
|
+
mutants: @inventory,
|
|
157
|
+
skipped: @skipped + suppression_entries,
|
|
158
|
+
totals: {
|
|
159
|
+
mutants: lists.sum(&:size),
|
|
160
|
+
dropped_by_max_mutants_per_method: @diagnostics.omissions.sum(&:count),
|
|
161
|
+
incomplete_files: incomplete
|
|
162
|
+
},
|
|
163
|
+
estimated_seconds: incomplete.zero? ? estimated_seconds(subjects, lists) : nil,
|
|
164
|
+
warnings: @diagnostics.warnings
|
|
165
|
+
))
|
|
166
|
+
nil
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def estimated_seconds(subjects, lists)
|
|
170
|
+
map = cached_map
|
|
171
|
+
seconds = map && work_seconds(map, subjects, lists)
|
|
172
|
+
seconds&.round(2)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def subject_entry(subject, list)
|
|
176
|
+
{ id: subject.id, file: relative_path(subject.file), start_line: subject.start_line,
|
|
177
|
+
end_line: subject.end_line, mutants: list.size }
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Byte offsets index the file as read from disk, like the Stryker
|
|
181
|
+
# report; original and replacement text make each edit self-contained.
|
|
182
|
+
def mutant_entry(subject, mutation)
|
|
183
|
+
{
|
|
184
|
+
digest: mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source),
|
|
185
|
+
subject: subject.id,
|
|
186
|
+
operator: mutation.operator.to_s,
|
|
187
|
+
file: relative_path(subject.file),
|
|
188
|
+
line: subject.line_of(mutation.edits.first.start_byte),
|
|
189
|
+
edits: mutation.edits.map do |edit|
|
|
190
|
+
{ start_byte: edit.start_byte, end_byte: edit.end_byte,
|
|
191
|
+
original: source_span(subject, edit), replacement: edit.replacement }
|
|
192
|
+
end
|
|
193
|
+
}
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def record_skip(subject, mutation, reason)
|
|
197
|
+
@skipped << mutant_entry(subject, mutation).merge(reason: reason)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Inline-suppressed and hazard-refused candidates with their reasons,
|
|
201
|
+
# recorded during candidate filtering, before diff restriction and the
|
|
202
|
+
# per-method cap (the same scope the text audit names).
|
|
203
|
+
def suppression_entries
|
|
204
|
+
inline = @diagnostics.inline_suppressions.map do |entry|
|
|
205
|
+
{ digest: entry[:digest], subject: entry[:subject], operator: entry[:operator].to_s,
|
|
206
|
+
file: relative_path(entry[:file]), line: entry[:line], reason: "inline_suppression: #{entry[:reason]}" }
|
|
207
|
+
end
|
|
208
|
+
hazards = @diagnostics.generation_events.select { |event| event.source == "hazard_guard" }.map do |event|
|
|
209
|
+
{ subject: event.subject, operator: event.operator.to_s, file: relative_path(event.file),
|
|
210
|
+
reason: "hazardous" }
|
|
211
|
+
end
|
|
212
|
+
inline + hazards
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def relative_path(file)
|
|
216
|
+
file.delete_prefix("#{@config.root}/")
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def source_span(subject, edit)
|
|
220
|
+
subject.file_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte).to_s
|
|
221
|
+
.force_encoding(Encoding::UTF_8)
|
|
222
|
+
end
|
|
114
223
|
end
|
|
115
224
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# The lexical `using` sites of one parsed file (#634). A bare `using`
|
|
6
|
+
# activates its refinement from where it sits to the end of its own
|
|
7
|
+
# scope: a class or module body, or the whole file at the top level.
|
|
8
|
+
# Activation is lexical, not namespaced: a `class ::Q` nested inside
|
|
9
|
+
# the body still sees it, while a reopened body with the same constant
|
|
10
|
+
# path after the scope's `end` does not. Sites therefore compare byte
|
|
11
|
+
# ranges, not constant paths: a site covers a subject that starts after
|
|
12
|
+
# the `using` and before the scope's closing `end`. A `using` inside a
|
|
13
|
+
# method or block body cannot be scoped this way, so one anywhere keeps
|
|
14
|
+
# the old whole-file skip. Skipping only ever means "not mutated",
|
|
15
|
+
# never a wrong verdict.
|
|
16
|
+
class RefinementActivations
|
|
17
|
+
Site = Data.define(:start_byte, :scope_end, :dynamic_scope) do
|
|
18
|
+
def covers?(subject)
|
|
19
|
+
!dynamic_scope && subject.start_byte > start_byte && subject.start_byte < scope_end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.mark(subjects, program)
|
|
24
|
+
sites = []
|
|
25
|
+
collect(program, program.end_offset, false, sites)
|
|
26
|
+
return if sites.empty?
|
|
27
|
+
|
|
28
|
+
if sites.any?(&:dynamic_scope)
|
|
29
|
+
subjects.each { |subject| subject.skip_reason = "refinements" }
|
|
30
|
+
return
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
subjects.each do |subject|
|
|
34
|
+
subject.skip_reason = "refinements" if sites.any? { |site| site.covers?(subject) }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.collect(node, scope_end, dynamic, sites)
|
|
39
|
+
if node.is_a?(Prism::CallNode) && node.name == :using && node.receiver.nil?
|
|
40
|
+
sites << Site.new(start_byte: node.location.start_offset, scope_end: scope_end, dynamic_scope: dynamic)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
descend(node, scope_end, dynamic, sites)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def self.descend(node, scope_end, dynamic, sites)
|
|
47
|
+
case node
|
|
48
|
+
when Prism::ClassNode
|
|
49
|
+
collect(node.superclass, scope_end, dynamic, sites) if node.superclass
|
|
50
|
+
collect(node.body, node.end_offset, dynamic, sites) if node.body
|
|
51
|
+
when Prism::ModuleNode, Prism::SingletonClassNode
|
|
52
|
+
collect(node.body, node.end_offset, dynamic, sites) if node.body
|
|
53
|
+
when Prism::DefNode, Prism::BlockNode, Prism::LambdaNode
|
|
54
|
+
node.compact_child_nodes.each { |child| collect(child, scope_end, true, sites) }
|
|
55
|
+
else
|
|
56
|
+
node.compact_child_nodes.each { |child| collect(child, scope_end, dynamic, sites) }
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -32,8 +32,16 @@ module Saboteur
|
|
|
32
32
|
|
|
33
33
|
# Where a verdict comes from: the exact selection and the state of the working
|
|
34
34
|
# source before the run started. subjects, tests and sources are arrays.
|
|
35
|
+
# typed_settings: the settings keys typed on the command line, so the
|
|
36
|
+
# generated commands can tell a typed empty list (which may clear a
|
|
37
|
+
# config-file value) from one the config itself supplies.
|
|
35
38
|
Provenance = Data.define(:ref, :seed, :head, :working_tree, :changed_files, :mode, :json_report, :subjects,
|
|
36
|
-
:tests, :sources, :config_path, :digest, :cache, :incremental, :settings
|
|
39
|
+
:tests, :sources, :config_path, :digest, :cache, :incremental, :settings,
|
|
40
|
+
:typed_settings) do
|
|
41
|
+
def initialize(typed_settings: [], **fields)
|
|
42
|
+
super
|
|
43
|
+
end
|
|
44
|
+
end
|
|
37
45
|
|
|
38
46
|
class << self
|
|
39
47
|
def call(report, provenance)
|
|
@@ -8,15 +8,27 @@ module Saboteur
|
|
|
8
8
|
module Agent
|
|
9
9
|
# The shell commands an agent is told to run next. Every word is escaped with
|
|
10
10
|
# Shellwords, so a ref, path or subject holding shell syntax stays one literal
|
|
11
|
-
# argument. A command is never cut short: when it is
|
|
12
|
-
# is left out, because half a command could run
|
|
11
|
+
# argument. A command for the agent protocol is never cut short: when it is
|
|
12
|
+
# longer than MAX_BYTES it is left out, because half a command could run
|
|
13
|
+
# something else. The text footer passes capped: false instead - there the
|
|
14
|
+
# promise is the complete command, however long (#261).
|
|
13
15
|
module Commands
|
|
14
16
|
MAX_BYTES = 1024
|
|
15
17
|
|
|
16
18
|
# Options typed on the command line that change how a run behaves.
|
|
17
19
|
SETTING_FLAGS = { reeval: "--reeval", integration: "--integration", jobs: "--jobs",
|
|
18
20
|
timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html",
|
|
19
|
-
operators: "--operators", exclude_operators: "--exclude-operators"
|
|
21
|
+
operators: "--operators", exclude_operators: "--exclude-operators",
|
|
22
|
+
baseline_passes: "--baseline-passes", max_duration: "--max-duration",
|
|
23
|
+
shard: "--shard" }.freeze
|
|
24
|
+
|
|
25
|
+
# CLI-only boolean switches recorded in settings, mapped by value, so
|
|
26
|
+
# an explicit false (--no-max-duration-ok over a config true) survives
|
|
27
|
+
# into the generated commands instead of reverting to the config.
|
|
28
|
+
BOOLEAN_SETTING_FLAGS = { allow_hazardous: { true => "--allow-hazardous" },
|
|
29
|
+
allow_focus: { true => "--allow-focus" },
|
|
30
|
+
max_duration_ok: { true => "--max-duration-ok",
|
|
31
|
+
false => "--no-max-duration-ok" } }.freeze
|
|
20
32
|
|
|
21
33
|
class << self
|
|
22
34
|
# Run the same selection again: ref, subjects, sources, tests, config,
|
|
@@ -26,45 +38,65 @@ module Saboteur
|
|
|
26
38
|
finish(words, provenance)
|
|
27
39
|
end
|
|
28
40
|
|
|
29
|
-
# Replay one survivor by digest in the same selection.
|
|
30
|
-
|
|
41
|
+
# Replay one survivor by digest in the same selection. capped: false
|
|
42
|
+
# skips the protocol byte cap, for callers that print to a terminal.
|
|
43
|
+
def replay(provenance, digest, capped: true)
|
|
31
44
|
words = ["saboteur", "run", "--id", digest, *selection(provenance, replay: true), "--seed",
|
|
32
45
|
provenance.seed.to_s]
|
|
33
|
-
finish(words, provenance)
|
|
46
|
+
finish(words, provenance, capped: capped)
|
|
34
47
|
end
|
|
35
48
|
|
|
36
49
|
private
|
|
37
50
|
|
|
38
|
-
def finish(words, provenance)
|
|
51
|
+
def finish(words, provenance, capped: true)
|
|
39
52
|
words += ["--", *provenance.sources] unless provenance.sources.empty?
|
|
40
53
|
command = Shellwords.join(words)
|
|
41
|
-
command
|
|
54
|
+
command if !capped || command.bytesize <= MAX_BYTES
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def mode_words(provenance)
|
|
58
|
+
words = []
|
|
59
|
+
words << "--no-cache" if provenance.cache == false
|
|
60
|
+
words << "--incremental" if provenance.incremental
|
|
61
|
+
words.concat(boolean_words(provenance))
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def boolean_words(provenance)
|
|
65
|
+
BOOLEAN_SETTING_FLAGS.filter_map { |key, words| words[provenance.settings[key]] }
|
|
42
66
|
end
|
|
43
67
|
|
|
44
|
-
# An explicit scope is frozen
|
|
45
|
-
#
|
|
68
|
+
# An explicit scope is frozen as --test, which wins over inferred
|
|
69
|
+
# selection by precedence, whatever the config file sets.
|
|
46
70
|
def test_words(tests)
|
|
47
|
-
tests.
|
|
71
|
+
tests.flat_map { |test| ["--test", test] }
|
|
48
72
|
end
|
|
49
73
|
|
|
50
|
-
|
|
74
|
+
# An empty list nobody typed came from the config file or its
|
|
75
|
+
# default, which the command reads again, so its flag is left out
|
|
76
|
+
# (#317). A typed empty list stays: it may clear a config value.
|
|
77
|
+
def setting_words(provenance, replay: false)
|
|
51
78
|
SETTING_FLAGS.flat_map do |key, flag|
|
|
79
|
+
next [] if replay && key == :shard
|
|
52
80
|
next [] unless provenance.settings.key?(key)
|
|
53
81
|
|
|
54
82
|
value = provenance.settings[key]
|
|
83
|
+
next [] if value == [] && !provenance.typed_settings.include?(key)
|
|
84
|
+
|
|
55
85
|
[flag, value.is_a?(Array) ? value.join(",") : value.to_s]
|
|
56
86
|
end
|
|
57
87
|
end
|
|
58
88
|
|
|
89
|
+
# Whole-suite scope is a bare --all, only when no tests resolved;
|
|
90
|
+
# --all never combines with --test.
|
|
59
91
|
def selection(provenance, replay: false)
|
|
60
92
|
words = []
|
|
61
93
|
words.push("--config", provenance.config_path) if provenance.config_path
|
|
62
94
|
words.push("--since", provenance.ref) if provenance.ref
|
|
63
95
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
96
|
+
words << "--all" if provenance.settings[:all] && provenance.tests.empty?
|
|
64
97
|
words.push(*test_words(provenance.tests))
|
|
65
|
-
words.concat(setting_words(provenance))
|
|
66
|
-
words
|
|
67
|
-
words << "--incremental" if provenance.incremental
|
|
98
|
+
words.concat(setting_words(provenance, replay: replay))
|
|
99
|
+
words.concat(mode_words(provenance))
|
|
68
100
|
words.push("--id", provenance.digest) if provenance.digest && !replay
|
|
69
101
|
words
|
|
70
102
|
end
|
|
@@ -8,10 +8,13 @@ module Saboteur
|
|
|
8
8
|
# stable code, a count and a plain meaning.
|
|
9
9
|
module Incomplete
|
|
10
10
|
NO_COVERAGE = "no coverage"
|
|
11
|
+
SYNTAX_FAILURE = "invalid_file_syntax"
|
|
12
|
+
UNREADABLE_FAILURE = "unreadable"
|
|
11
13
|
|
|
12
14
|
class << self
|
|
13
15
|
def reasons(report)
|
|
14
16
|
reasons = []
|
|
17
|
+
shard_reason(reasons, report)
|
|
15
18
|
add(reasons, "no_coverage", skipped_for_coverage(report), "subjects or mutants had no covering test")
|
|
16
19
|
load_only = report.counts[:ran_at_load] + report.skipped_subjects.values.count do |reason|
|
|
17
20
|
reason == Selection::CoverageMap::RAN_AT_LOAD
|
|
@@ -23,14 +26,47 @@ module Saboteur
|
|
|
23
26
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
24
27
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
25
28
|
add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
|
|
26
|
-
add(reasons, "
|
|
27
|
-
"
|
|
29
|
+
add(reasons, "unverified", report.counts[:unverified],
|
|
30
|
+
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
31
|
+
discovery_failures(reasons, report)
|
|
32
|
+
early_stop_reason(reasons, report)
|
|
28
33
|
inline_scope(reasons, report)
|
|
29
34
|
reasons
|
|
30
35
|
end
|
|
31
36
|
|
|
32
37
|
private
|
|
33
38
|
|
|
39
|
+
# A shard always runs a subset of the planned inventory by contract, even
|
|
40
|
+
# when this shard happened to select every planned mutant (count 0).
|
|
41
|
+
def shard_reason(reasons, report)
|
|
42
|
+
shard = report.shard
|
|
43
|
+
return unless shard
|
|
44
|
+
|
|
45
|
+
reasons << { reason: "shard", count: shard.planned_count - shard.selected_count,
|
|
46
|
+
meaning: "planned mutants assigned to other shards" }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Source the selection could not read at all: its subjects never
|
|
50
|
+
# entered the run, so a clean or complete verdict would be wrong.
|
|
51
|
+
def discovery_failures(reasons, report)
|
|
52
|
+
failures = report.discovery_failures
|
|
53
|
+
add(reasons, SYNTAX_FAILURE, failures.count { |f| f.reason == SYNTAX_FAILURE },
|
|
54
|
+
"included files did not parse; their subjects were not discovered")
|
|
55
|
+
add(reasons, UNREADABLE_FAILURE, failures.count { |f| f.reason == UNREADABLE_FAILURE },
|
|
56
|
+
"included untracked files could not be read; their subjects were not covered")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def early_stop_reason(reasons, report)
|
|
60
|
+
stop = report.early_stop
|
|
61
|
+
return unless stop
|
|
62
|
+
|
|
63
|
+
label = stop.reason == "fail_fast" ? "--fail-fast stopped" : "the --max-duration budget stopped"
|
|
64
|
+
add(reasons, stop.reason, stop.unreached.size, "selected subjects did not run because #{label} the run")
|
|
65
|
+
omitted = stop.truncated.sum { |entry| entry[:omitted] }
|
|
66
|
+
add(reasons, "#{stop.reason}_truncated", omitted,
|
|
67
|
+
"mutants omitted inside subjects that stopped mid-way because #{label} the run")
|
|
68
|
+
end
|
|
69
|
+
|
|
34
70
|
def inline_scope(reasons, report)
|
|
35
71
|
add(reasons, "inline_suppressed", report.inline_suppressions.size,
|
|
36
72
|
"candidates were suppressed by inline directives; see the full report audit")
|
|
@@ -26,10 +26,11 @@ module Saboteur
|
|
|
26
26
|
counts: counts(report), score: score(report),
|
|
27
27
|
scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
|
|
28
28
|
incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
|
|
29
|
-
findings: findings(report, shown, snippet_limit)
|
|
29
|
+
findings: findings(report, shown, snippet_limit)
|
|
30
|
+
}.merge(report.shard ? { shard: report.shard.to_h } : {}).merge(
|
|
30
31
|
warnings: report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] },
|
|
31
32
|
next: next_steps(report, provenance)
|
|
32
|
-
|
|
33
|
+
)
|
|
33
34
|
end
|
|
34
35
|
|
|
35
36
|
private
|
|
@@ -95,7 +96,7 @@ module Saboteur
|
|
|
95
96
|
|
|
96
97
|
def shown_findings(listed, shown, snippet_limit)
|
|
97
98
|
listed.first(shown).map do |finding|
|
|
98
|
-
finding.slice(:digest, :line, :endLine, :operator, :coarse)
|
|
99
|
+
finding.slice(:digest, :line, :endLine, :operator, :coarse, :classification)
|
|
99
100
|
.merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
|
|
100
101
|
original: clip(finding[:original], snippet_limit),
|
|
101
102
|
replacement: clip(finding[:replacement], snippet_limit))
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "survivor_ranking"
|
|
5
|
+
require_relative "survivor_classification"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
module Reporter
|
|
@@ -59,8 +60,10 @@ module Saboteur
|
|
|
59
60
|
original: joined_texts(result) { |edit| original_text(result, edit) },
|
|
60
61
|
replacement: joined_texts(result, &:replacement),
|
|
61
62
|
edits: entries,
|
|
62
|
-
coarse: result.mutation.coarse
|
|
63
|
-
|
|
63
|
+
coarse: result.mutation.coarse,
|
|
64
|
+
# Why the survivor is worth a look, as a label only (#192).
|
|
65
|
+
classification: SurvivorClassification.label(result)
|
|
66
|
+
}.compact
|
|
64
67
|
entry = with_changed_lines(report, result, entry)
|
|
65
68
|
result.ignored ? entry.merge(reason: result.ignored) : entry
|
|
66
69
|
end
|
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
module Saboteur
|
|
5
5
|
module Reporter
|
|
6
6
|
# v1 stays readable, but v2 records require versioned operator evidence.
|
|
7
|
+
# A policy stamped with an older POLICY_VERSION is shape-checked only:
|
|
8
|
+
# the operator set has moved on, so it never recomputes equal - and it
|
|
9
|
+
# never compares equal to a current run's policy either, which is what
|
|
10
|
+
# keeps trend from mixing operator sets (#135).
|
|
7
11
|
module HistoryPolicy
|
|
8
12
|
def validate_header(record, fields)
|
|
9
13
|
version = record.is_a?(Hash) && record["version"]
|
|
@@ -18,6 +22,11 @@ module Saboteur
|
|
|
18
22
|
raise Error, "score history contains invalid operator policy" unless value.is_a?(Hash)
|
|
19
23
|
|
|
20
24
|
fields(value, %w[version selected excluded effective])
|
|
25
|
+
policy_version = value["version"]
|
|
26
|
+
unless policy_version.is_a?(Integer) &&
|
|
27
|
+
policy_version.between?(1, MutationSource::OperatorRegistry::POLICY_VERSION)
|
|
28
|
+
raise Error, "score history contains invalid operator policy"
|
|
29
|
+
end
|
|
21
30
|
raise Error, "score history contains invalid operator policy" if value["effective"].empty?
|
|
22
31
|
|
|
23
32
|
lists = value.values_at("selected", "excluded", "effective")
|
|
@@ -25,6 +34,8 @@ module Saboteur
|
|
|
25
34
|
raise Error, "score history contains invalid operator policy"
|
|
26
35
|
end
|
|
27
36
|
|
|
37
|
+
return if policy_version != MutationSource::OperatorRegistry::POLICY_VERSION
|
|
38
|
+
|
|
28
39
|
expected = MutationSource::OperatorRegistry.policy(value["selected"], value["excluded"])
|
|
29
40
|
raise Error, "score history contains invalid operator policy" unless value == expected
|
|
30
41
|
rescue ConfigError, NoMethodError, TypeError
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "score_history"
|
|
5
|
+
require_relative "history_record"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Reporter
|
|
9
|
+
# Read side of the append-only score history for `saboteur badge` and
|
|
10
|
+
# `saboteur trend` (#474). Every record is re-validated on read and the
|
|
11
|
+
# file discipline mirrors ScoreHistory: regular file, no symlink,
|
|
12
|
+
# byte-capped lines, no repair of imported evidence.
|
|
13
|
+
module HistoryReader
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def read(path)
|
|
17
|
+
records = []
|
|
18
|
+
File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |file|
|
|
19
|
+
raise Error, "score history must be a regular file" unless file.stat.file?
|
|
20
|
+
|
|
21
|
+
file.flock(File::LOCK_SH)
|
|
22
|
+
limit = ScoreHistory::MAX_RECORD_BYTES
|
|
23
|
+
while (line = file.gets(limit + 1))
|
|
24
|
+
raise Error, "score history record exceeds byte limit" if line.bytesize > limit
|
|
25
|
+
raise Error, "score history contains a truncated record" unless line.end_with?("\n")
|
|
26
|
+
|
|
27
|
+
records << HistoryRecord.parse(line)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
records
|
|
31
|
+
rescue SystemCallError, IOError => e
|
|
32
|
+
raise Error, "score history cannot be read: #{e.message}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The ScoreHistory change rules: same record version, selection,
|
|
36
|
+
# inventory and operator policy (a null policy never compares), both
|
|
37
|
+
# runs complete.
|
|
38
|
+
def comparable?(previous, current)
|
|
39
|
+
previous["version"] == current["version"] &&
|
|
40
|
+
previous["selection"] == current["selection"] &&
|
|
41
|
+
previous["inventory"] == current["inventory"] &&
|
|
42
|
+
previous["incomplete"].empty? && current["incomplete"].empty? &&
|
|
43
|
+
!previous["operatorPolicy"].nil? && previous["operatorPolicy"] == current["operatorPolicy"]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|