saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
require "cgi"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
|
|
7
|
+
require_relative "survivor_classification"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
module Reporter
|
|
9
11
|
# One self-contained HTML page: inline style, no script, no network. It is
|
|
@@ -23,7 +25,9 @@ module Saboteur
|
|
|
23
25
|
.status{border-radius:3px;padding:0 .4rem;color:#fff;white-space:nowrap}
|
|
24
26
|
.Killed{background:#2e7d32}.Survived{background:#c62828}.Timeout{background:#b45309}
|
|
25
27
|
.RuntimeError{background:#6a1b9a}.NoCoverage{background:#455a64}.Ignored{background:#616161}
|
|
26
|
-
.Accepted{background:#8d6e63}
|
|
28
|
+
.Accepted,.Triage_accepted_equivalent{background:#8d6e63}
|
|
29
|
+
.Triage_wont_fix{background:#616161}.Triage_todo{background:#b45309}
|
|
30
|
+
.classification{border-radius:3px;padding:0 .4rem;background:#eceff1;color:#37474f;white-space:nowrap}
|
|
27
31
|
.warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
|
|
28
32
|
code,pre{font-family:ui-monospace,monospace;font-size:13px}
|
|
29
33
|
pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
|
|
@@ -195,12 +199,30 @@ module Saboteur
|
|
|
195
199
|
def row_status(mutant, result)
|
|
196
200
|
badge = status_label(mutant[:status])
|
|
197
201
|
# Only survivors are ever put on the ignore list.
|
|
198
|
-
result.ignored
|
|
202
|
+
if result.ignored || result.triage_state
|
|
203
|
+
# An entry without a state predates triage states: render Accepted.
|
|
204
|
+
badge = if result.triage_state.nil?
|
|
205
|
+
"#{badge} #{status_label('Accepted')}"
|
|
206
|
+
else
|
|
207
|
+
"#{badge} #{status_label(triage_badge(result), css_class: "Triage_#{result.triage_state}")}"
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
# The survivor's classification, as a label only (#192).
|
|
211
|
+
label = SurvivorClassification.label(result)
|
|
212
|
+
label ? "#{badge} <span class=\"classification\">#{escape(label)}</span>" : badge
|
|
199
213
|
end
|
|
200
214
|
|
|
201
215
|
# One edit shows the original text and its replacement. A mutant with
|
|
202
216
|
# several edits shows only the replacements, because one location cannot
|
|
203
217
|
# stand for all of them.
|
|
218
|
+
def triage_badge(result)
|
|
219
|
+
case result.triage_state
|
|
220
|
+
when "wont_fix" then "Won\u2019t fix"
|
|
221
|
+
when "todo" then "Todo"
|
|
222
|
+
else "Accepted"
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
204
226
|
def change(mutant, result)
|
|
205
227
|
edits = result.mutation.edits
|
|
206
228
|
return "<pre>#{escape(mutant[:replacement])}</pre>" unless edits.size == 1
|
|
@@ -212,8 +234,8 @@ module Saboteur
|
|
|
212
234
|
"<pre>#{escape(edit.replacement)}</pre>"
|
|
213
235
|
end
|
|
214
236
|
|
|
215
|
-
def status_label(status)
|
|
216
|
-
%(<span class="status #{
|
|
237
|
+
def status_label(status, css_class: status)
|
|
238
|
+
%(<span class="status #{css_class}">#{status}</span>)
|
|
217
239
|
end
|
|
218
240
|
|
|
219
241
|
def escape(text)
|
|
@@ -20,6 +20,12 @@ module Saboteur
|
|
|
20
20
|
join(result, &:replacement)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# A mutant that only deletes: every edit's replacement is empty.
|
|
24
|
+
# Rendered as "removes X" rather than "X -> " with a blank side.
|
|
25
|
+
def removal?(result)
|
|
26
|
+
sorted_edits(result).all? { |edit| edit.replacement.empty? }
|
|
27
|
+
end
|
|
28
|
+
|
|
23
29
|
def sorted_edits(result)
|
|
24
30
|
result.mutation.edits.sort_by(&:start_byte)
|
|
25
31
|
end
|
|
@@ -7,9 +7,14 @@ module Saboteur
|
|
|
7
7
|
# accept one, and how to hand the run to an agent. Silent when nothing
|
|
8
8
|
# survived, so a clean run stays short.
|
|
9
9
|
class NextSteps
|
|
10
|
-
|
|
10
|
+
# replay_command: digest -> the full, copy-pasteable replay command
|
|
11
|
+
# carrying the run's own selection options, however long it gets
|
|
12
|
+
# (#261). When no builder is wired the digest-only hint (with its
|
|
13
|
+
# reassembly note) is the fallback.
|
|
14
|
+
def initialize(out, palette, replay_command: nil)
|
|
11
15
|
@out = out
|
|
12
16
|
@palette = palette
|
|
17
|
+
@replay_command = replay_command
|
|
13
18
|
end
|
|
14
19
|
|
|
15
20
|
def call(report)
|
|
@@ -17,9 +22,14 @@ module Saboteur
|
|
|
17
22
|
return unless survivor
|
|
18
23
|
|
|
19
24
|
@out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
25
|
+
command = @replay_command&.call(survivor.digest)
|
|
26
|
+
if command
|
|
27
|
+
@out.puts " replay one survivor: #{command}"
|
|
28
|
+
else
|
|
29
|
+
seed = " --seed #{report.seed}" if report.seed
|
|
30
|
+
@out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
|
|
31
|
+
@out.puts " (add this run's own --config, --since, --subject, --test and source options)"
|
|
32
|
+
end
|
|
23
33
|
@out.puts " accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
|
|
24
34
|
entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
|
|
25
35
|
@out.puts " #{entry}"
|
|
@@ -39,7 +39,11 @@ module Saboteur
|
|
|
39
39
|
file.flush
|
|
40
40
|
change(previous, record)
|
|
41
41
|
end
|
|
42
|
-
|
|
42
|
+
# SystemCallError/IOError from the write itself propagate unwrapped on
|
|
43
|
+
# purpose: the CLI treats those as a soft artifact failure (warn and
|
|
44
|
+
# exit by the results), while validation, integrity, record-size and
|
|
45
|
+
# lock failures stay hard errors (#296).
|
|
46
|
+
rescue JSON::ParserError, KeyError, TypeError => e
|
|
43
47
|
raise Error, "score history was not recorded: #{e.message}"
|
|
44
48
|
end
|
|
45
49
|
|
|
@@ -39,12 +39,24 @@ module Saboteur
|
|
|
39
39
|
rejectedKills: report.rejected_kills,
|
|
40
40
|
warnings: report.warnings,
|
|
41
41
|
framework: { name: "saboteur", version: VERSION },
|
|
42
|
-
saboteur:
|
|
43
|
-
inlineSuppressions: inline_audit(report) }
|
|
44
|
-
.merge(operatorPolicy: report.operator_policy).merge(exit_cause ? { exitCause: exit_cause } : {})
|
|
42
|
+
saboteur: saboteur_enrichment(report, exit_cause)
|
|
45
43
|
}.merge(report.seed ? { config: { seed: report.seed } } : {})
|
|
46
44
|
end
|
|
47
45
|
|
|
46
|
+
# saboteur-namespaced keys beyond Stryker's schema: findings and
|
|
47
|
+
# scores, then the optional policy, shard, exit-cause and memory keys.
|
|
48
|
+
def self.saboteur_enrichment(report, exit_cause)
|
|
49
|
+
{ findings: Findings.build(report), scores: scores(report),
|
|
50
|
+
inlineSuppressions: inline_audit(report) }
|
|
51
|
+
.merge(operatorPolicy: report.operator_policy).merge(shard(report))
|
|
52
|
+
.merge(exit_cause ? { exitCause: exit_cause } : {})
|
|
53
|
+
.merge(report.memory ? { memory: report.memory } : {})
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.shard(report)
|
|
57
|
+
report.shard ? { shard: report.shard.to_h } : {}
|
|
58
|
+
end
|
|
59
|
+
|
|
48
60
|
def self.inline_audit(report)
|
|
49
61
|
report.inline_suppressions.map { |entry| entry.merge(file: relative(entry[:file], report.root)) }
|
|
50
62
|
end
|
|
@@ -58,7 +70,14 @@ module Saboteur
|
|
|
58
70
|
# nil for a run that covered its whole selection.
|
|
59
71
|
def self.early_stop(report)
|
|
60
72
|
stop = report.early_stop
|
|
61
|
-
|
|
73
|
+
return unless stop
|
|
74
|
+
|
|
75
|
+
{ reason: stop.reason, unreachedSubjects: stop.unreached, truncatedSubjects: truncated_subjects(stop),
|
|
76
|
+
unverifiedSubjects: stop.unverified }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.truncated_subjects(stop)
|
|
80
|
+
stop.truncated.map { |entry| { subject: entry[:subject], omittedMutants: entry[:omitted] } }
|
|
62
81
|
end
|
|
63
82
|
|
|
64
83
|
def self.skipped_subjects(report)
|
|
@@ -100,7 +119,14 @@ module Saboteur
|
|
|
100
119
|
|
|
101
120
|
def self.optional_fields(result, test_ids)
|
|
102
121
|
{
|
|
122
|
+
# A rejected kill is a verdict, not a tool error: the tests failed
|
|
123
|
+
# but also fail with the method unchanged. Flag it per mutant;
|
|
124
|
+
# the top-level rejectedKills stays a bare count (#375).
|
|
125
|
+
rejectedKill: (true if result.rejected_kill?),
|
|
103
126
|
statusReason: status_reason(result),
|
|
127
|
+
# Why the survivor is worth a look, as a label only: scores and
|
|
128
|
+
# the exit code never change (#192).
|
|
129
|
+
classification: SurvivorClassification.label(result),
|
|
104
130
|
killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
|
|
105
131
|
coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
|
|
106
132
|
duration: result.runtime && (result.runtime * 1000).round
|
|
@@ -115,17 +141,24 @@ module Saboteur
|
|
|
115
141
|
when :error, :resource_limit then "RuntimeError"
|
|
116
142
|
when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
|
|
117
143
|
# The schema has no status for "ran, but no test can observe a
|
|
118
|
-
# change here"
|
|
119
|
-
|
|
144
|
+
# change here", nor for a verdict the budget left unchecked.
|
|
145
|
+
# Ignored with the reason is the honest mapping.
|
|
146
|
+
when :ran_at_load, :unreachable, :unverified then "Ignored"
|
|
120
147
|
end
|
|
121
148
|
end
|
|
122
149
|
|
|
123
150
|
def self.status_reason(result)
|
|
151
|
+
return "todo: #{result.triage_reason}" if result.triage_state == "todo"
|
|
152
|
+
# The "ignored: " prefix is a public contract consumers classify by
|
|
153
|
+
# (#304 CI): accepted_equivalent stays byte-identical to a plain
|
|
154
|
+
# ignore, and only wont_fix adds a suffix after the reason.
|
|
155
|
+
return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
|
|
124
156
|
return "ignored: #{result.ignored}" if result.ignored
|
|
125
157
|
return "resource limit: #{result.reason}" if result.status == :resource_limit
|
|
126
158
|
return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
|
|
127
159
|
|
|
128
|
-
result.reason.to_s.
|
|
160
|
+
reason = result.reason.to_s.strip
|
|
161
|
+
reason.empty? ? nil : reason
|
|
129
162
|
end
|
|
130
163
|
|
|
131
164
|
def self.location(edit, source)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../engine/bytecode_equivalence"
|
|
5
|
+
require_relative "../execution/external_runner"
|
|
6
|
+
require_relative "../mutation_source/builtin"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module Reporter
|
|
10
|
+
# Why a survivor is worth a look, as a LABEL ONLY (#192): the label
|
|
11
|
+
# lands in reports so a reader can sort real gaps from noise. Every
|
|
12
|
+
# label is a HEURISTIC, not a diagnosis - inspect the survivor before
|
|
13
|
+
# acting on it, and never add a test or ignore a mutant from the label
|
|
14
|
+
# alone. Labels never change scores or the exit code.
|
|
15
|
+
#
|
|
16
|
+
# One label per survivor, first match wins:
|
|
17
|
+
# likely_equivalent - the mutant compiles to the same instructions as
|
|
18
|
+
# the original (the #151/#169 bytecode filter): strong evidence of
|
|
19
|
+
# a no-op mutant, worth checking against the source. First: a
|
|
20
|
+
# no-op mutant's survival says nothing about the suite.
|
|
21
|
+
# arid - every line an edit touches matches an arid pattern
|
|
22
|
+
# (logging and friends, #111/#187). The builtin hint list decides;
|
|
23
|
+
# a configured arid policy does not change the label.
|
|
24
|
+
# missing_test - a test covered the line and passed, which MAY mean
|
|
25
|
+
# the suite reaches the code without pinning this behaviour. The
|
|
26
|
+
# covering list is only the PLANNED coverage: whole-body (coarse)
|
|
27
|
+
# mutants carry method-level guesses, so they never get this
|
|
28
|
+
# label, and neither does anything without real coverage.
|
|
29
|
+
# tool_limit - reserved for survivors saboteur could not observe
|
|
30
|
+
# faithfully (re-eval refusal, external runner without coverage). No
|
|
31
|
+
# result records that signal yet, so the label is never emitted.
|
|
32
|
+
#
|
|
33
|
+
# A survivor with no matching signal stays unlabeled.
|
|
34
|
+
module SurvivorClassification
|
|
35
|
+
LABELS = %w[likely_equivalent arid missing_test tool_limit].freeze
|
|
36
|
+
|
|
37
|
+
class << self
|
|
38
|
+
def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
|
|
39
|
+
return nil unless result.status == :survived
|
|
40
|
+
return "likely_equivalent" if equivalent?(result)
|
|
41
|
+
return "arid" if arid?(result, arid)
|
|
42
|
+
return "missing_test" if possible_test_gap?(result)
|
|
43
|
+
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
# The mutant's own def text, compiled side by side with the
|
|
50
|
+
# original: equal instruction sequences mean behaviour cannot
|
|
51
|
+
# change. Edits index the whole file, so they shift by the def's
|
|
52
|
+
# start byte before splicing into the def text.
|
|
53
|
+
def equivalent?(result)
|
|
54
|
+
subject = result.subject
|
|
55
|
+
original = subject.source
|
|
56
|
+
mutated = result.mutation.edits.sort_by(&:start_byte).reverse.inject(original) do |text, edit|
|
|
57
|
+
from = edit.start_byte - subject.start_byte
|
|
58
|
+
to = edit.end_byte - subject.start_byte
|
|
59
|
+
text.byteslice(0, from) + edit.replacement + text.byteslice(to..)
|
|
60
|
+
end
|
|
61
|
+
Engine::BytecodeEquivalence.new.equivalent?(original, mutated,
|
|
62
|
+
frozen_string_literal: subject.frozen_literals)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# covered_by is a PLANNED covering list, not a verdict about what
|
|
66
|
+
# the suite asserts: coarse (whole-body) mutants carry method-level
|
|
67
|
+
# guesses, and a --test-command run records only the pseudo-test id
|
|
68
|
+
# for every mutant, so neither reads as real coverage (#305 QA).
|
|
69
|
+
def possible_test_gap?(result)
|
|
70
|
+
real_coverage = result.covered_by - [Execution::ExternalRunner::PSEUDO_TEST]
|
|
71
|
+
!result.mutation.coarse && real_coverage.any?
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Arid only when EVERY line an edit touches reads as arid: an edit
|
|
75
|
+
# that starts on a logging line but removes a composite body is not
|
|
76
|
+
# arid. The operator never decides: removing a logging statement is
|
|
77
|
+
# arid whatever removed it.
|
|
78
|
+
def arid?(result, patterns)
|
|
79
|
+
result.mutation.edits.all? do |edit|
|
|
80
|
+
lines = edited_lines(result.subject, edit)
|
|
81
|
+
lines.any? && lines.all? do |line|
|
|
82
|
+
patterns.any? { |pattern| line.strip.match?(Regexp.new(pattern)) }
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Edit offsets are BYTES, so the line lookup runs on the binary
|
|
88
|
+
# view (index/rindex are byte-exact there) and only the extracted
|
|
89
|
+
# lines are decoded, scrubbed so a partial multibyte sequence at an
|
|
90
|
+
# edit boundary cannot raise on the pattern match.
|
|
91
|
+
def edited_lines(subject, edit)
|
|
92
|
+
bytes = subject.file_source.b
|
|
93
|
+
last = [edit.end_byte - 1, edit.start_byte].max
|
|
94
|
+
line_start = (bytes.byteslice(0, edit.start_byte).rindex("\n") || -1) + 1
|
|
95
|
+
line_end = bytes.index("\n", last) || bytes.bytesize
|
|
96
|
+
bytes.byteslice(line_start, line_end - line_start).force_encoding(Encoding::UTF_8).scrub.lines
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -19,6 +19,9 @@ module Saboteur
|
|
|
19
19
|
"safe_navigation" => "try an example where the receiver is nil",
|
|
20
20
|
"argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
|
|
21
21
|
"pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
|
|
22
|
+
"chain_removal" => "try the shorter chain's value against the longer one; if you confirm the " \
|
|
23
|
+
"shorter chain is equivalent across the intended inputs (for example a duplicated " \
|
|
24
|
+
"column), simplify the code or ignore the mutant",
|
|
22
25
|
"endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
|
|
23
26
|
"filter_replacement" => "try items that match and items that do not, to tell select from reject"
|
|
24
27
|
}.freeze
|
|
@@ -47,14 +47,16 @@ module Saboteur
|
|
|
47
47
|
end
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
-
# Mutants on one line share the printed evidence
|
|
51
|
-
# same
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
50
|
+
# Mutants on one line share the printed evidence when it names the
|
|
51
|
+
# same tests: the covering-test list is compared as a set (#502), so
|
|
52
|
+
# one line's mutants no longer reprint an identical block only because
|
|
53
|
+
# the run measured the killer order per mutant. Reuse and changed
|
|
54
|
+
# lines still group exactly. Each run of neighbours with the same key
|
|
55
|
+
# is one group and shares evidence above its own mutations; evidence
|
|
56
|
+
# that repeats after another group is printed again, which keeps the
|
|
57
|
+
# ranked order. The block prints in the first mutant's run order.
|
|
56
58
|
def evidence_groups(report, on_line)
|
|
57
|
-
on_line.chunk { |result| [result.covered_by, result.reused?, report.changed_lines_for(result)] }
|
|
59
|
+
on_line.chunk { |result| [result.covered_by.uniq.sort, result.reused?, report.changed_lines_for(result)] }
|
|
58
60
|
.map(&:last)
|
|
59
61
|
end
|
|
60
62
|
|
|
@@ -70,9 +72,13 @@ module Saboteur
|
|
|
70
72
|
end
|
|
71
73
|
|
|
72
74
|
def mutation(result)
|
|
75
|
+
change = if MutationText.removal?(result)
|
|
76
|
+
"removes #{MutationText.original(result)}"
|
|
77
|
+
else
|
|
78
|
+
"#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))}"
|
|
79
|
+
end
|
|
73
80
|
text = " #{@palette.paint(:red, '-')} #{result.mutation.operator} " \
|
|
74
|
-
"#{
|
|
75
|
-
"#{result.digest}"
|
|
81
|
+
"#{change} #{result.digest}"
|
|
76
82
|
text += " (ignored: #{result.ignored})" if result.ignored
|
|
77
83
|
@out.puts text
|
|
78
84
|
hint(result)
|
|
@@ -17,12 +17,12 @@ module Saboteur
|
|
|
17
17
|
MAX_ERRORS = 10
|
|
18
18
|
MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
|
|
19
19
|
|
|
20
|
-
def initialize(out, verbose: false, color: :auto)
|
|
20
|
+
def initialize(out, verbose: false, color: :auto, replay_command: nil)
|
|
21
21
|
@out = out
|
|
22
22
|
@verbose = verbose
|
|
23
23
|
@palette = Palette.for(out, mode: color)
|
|
24
24
|
@score_table = ScoreTable.new(out, @palette)
|
|
25
|
-
@next_steps = NextSteps.new(out, @palette)
|
|
25
|
+
@next_steps = NextSteps.new(out, @palette, replay_command: replay_command)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def call(report)
|
|
@@ -91,6 +91,7 @@ module Saboteur
|
|
|
91
91
|
seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
|
|
92
92
|
@out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
|
|
93
93
|
"wall #{format('%.2f', report.wall_time)} s#{seed}"
|
|
94
|
+
@out.puts report.shard.line if report.shard
|
|
94
95
|
@out.puts status_line(report)
|
|
95
96
|
details(report)
|
|
96
97
|
@score_table.call(report)
|
|
@@ -103,12 +104,15 @@ module Saboteur
|
|
|
103
104
|
survived = "survived #{c[:survived]}"
|
|
104
105
|
ignored = report.survivors.size - report.unignored_survivors.size
|
|
105
106
|
survived += " (#{ignored} ignored)" if ignored.positive?
|
|
107
|
+
rejected = report.rejected_kills
|
|
108
|
+
rejected_note = " (#{rejected} rejected kill#{'s' if rejected > 1})" if rejected.positive?
|
|
106
109
|
line = "#{@palette.paint(:green,
|
|
107
110
|
"killed #{c[:killed]}")} #{@palette.paint(:red,
|
|
108
111
|
survived)} timeout #{c[:timeout]} " \
|
|
109
|
-
"resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
|
|
112
|
+
"resource limit #{c[:resource_limit]} error #{c[:error]}#{rejected_note} skipped #{c[:skipped]}"
|
|
110
113
|
line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
|
|
111
114
|
line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
|
|
115
|
+
line += " unverified #{c[:unverified]}" if c[:unverified].positive?
|
|
112
116
|
line
|
|
113
117
|
end
|
|
114
118
|
|
|
@@ -121,11 +125,19 @@ module Saboteur
|
|
|
121
125
|
|
|
122
126
|
def details(report)
|
|
123
127
|
breakdown = report.score_breakdown
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
128
|
+
# The share covers scored mutants only, so an error-dominated run
|
|
129
|
+
# must not print it as a bare score: "score 100.0 %" above "the run
|
|
130
|
+
# tested almost nothing" reads as a contradiction (#557).
|
|
131
|
+
if report.majority_errored?
|
|
132
|
+
@out.puts "score n/a (#{breakdown.tested} of #{report.executed.size} mutants scored, " \
|
|
133
|
+
"#{percent(breakdown.value)} of scored)"
|
|
134
|
+
else
|
|
135
|
+
@out.puts "score #{percent(breakdown.value)}" \
|
|
136
|
+
"#{if @verbose
|
|
137
|
+
"#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
|
|
138
|
+
'errors, resource limits and skips excluded'
|
|
139
|
+
end}"
|
|
140
|
+
end
|
|
129
141
|
report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
|
|
130
142
|
reused = report.results.count(&:reused?)
|
|
131
143
|
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
@@ -223,17 +235,51 @@ module Saboteur
|
|
|
223
235
|
end
|
|
224
236
|
end
|
|
225
237
|
|
|
226
|
-
# The verdict covers only the
|
|
238
|
+
# The verdict covers only the completed work, so say it where the
|
|
227
239
|
# score is read.
|
|
228
240
|
def early_stop(report)
|
|
229
241
|
stop = report.early_stop
|
|
230
242
|
return unless stop
|
|
231
243
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
"covers only
|
|
235
|
-
stop.
|
|
236
|
-
|
|
244
|
+
label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
|
|
245
|
+
@out.puts @palette.paint(:yellow, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
|
|
246
|
+
"covers only completed work")
|
|
247
|
+
budget_shortfall(stop, report.results.size)
|
|
248
|
+
list(stop.unreached) { |id| " not run: #{id}" }
|
|
249
|
+
list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
|
|
250
|
+
list(stop.unverified) { |id| " unverified: #{id}" }
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def list(entries)
|
|
254
|
+
entries.first(MAX_ERRORS).each { |entry| @out.puts yield(entry) }
|
|
255
|
+
@out.puts " ... #{entries.size - MAX_ERRORS} more" if entries.size > MAX_ERRORS
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def budget_shortfall(stop, completed)
|
|
259
|
+
return unless stop.reason == "max_duration"
|
|
260
|
+
|
|
261
|
+
line = if stop.planned_mutants
|
|
262
|
+
"partial: #{stop.planned_mutants - completed} of #{stop.planned_mutants} mutants not run"
|
|
263
|
+
else
|
|
264
|
+
"partial: mutant count unknown; the budget was spent before planning finished"
|
|
265
|
+
end
|
|
266
|
+
@out.puts @palette.paint(:yellow, line)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def early_stop_summary(stop)
|
|
270
|
+
parts = []
|
|
271
|
+
if stop.unreached.any?
|
|
272
|
+
parts << "#{stop.unreached.size} selected subject#{'s' unless stop.unreached.one?} did not run"
|
|
273
|
+
end
|
|
274
|
+
if stop.truncated.any?
|
|
275
|
+
omitted = stop.truncated.sum { |entry| entry[:omitted] }
|
|
276
|
+
parts << "#{stop.truncated.size} subject#{'s' unless stop.truncated.one?} stopped mid-way " \
|
|
277
|
+
"(#{omitted} mutants omitted)"
|
|
278
|
+
end
|
|
279
|
+
if stop.unverified.any?
|
|
280
|
+
parts << "#{stop.unverified.size} subject#{'s' unless stop.unverified.one?} left with unverified results"
|
|
281
|
+
end
|
|
282
|
+
parts.join("; ")
|
|
237
283
|
end
|
|
238
284
|
|
|
239
285
|
def warnings(report)
|
|
@@ -241,13 +287,24 @@ module Saboteur
|
|
|
241
287
|
end
|
|
242
288
|
|
|
243
289
|
def errors(report)
|
|
244
|
-
|
|
290
|
+
# Rejected kills are verdicts, not tool errors: the tests failed,
|
|
291
|
+
# but they also fail with the method unchanged (#375).
|
|
292
|
+
rejected, errored = report.results.select { |result| result.status == :error }.partition(&:rejected_kill?)
|
|
293
|
+
unless rejected.empty?
|
|
294
|
+
@out.puts "\nRejected kills (tests also fail with the method unchanged; not counted as killed):"
|
|
295
|
+
print_error_lines(rejected)
|
|
296
|
+
end
|
|
245
297
|
return if errored.empty?
|
|
246
298
|
|
|
247
299
|
@out.puts "\nErrors (not counted as killed):"
|
|
300
|
+
print_error_lines(errored)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def print_error_lines(errored)
|
|
248
304
|
errored.first(MAX_ERRORS).each do |result|
|
|
249
|
-
|
|
250
|
-
|
|
305
|
+
lines = result.reason.to_s.lines.map(&:strip)
|
|
306
|
+
@out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: #{lines.first}"
|
|
307
|
+
lines.drop(1).each { |line| @out.puts " #{line}" }
|
|
251
308
|
end
|
|
252
309
|
@out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
|
|
253
310
|
end
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -12,7 +12,10 @@ module Saboteur
|
|
|
12
12
|
# test can observe a change there, so it is outside the score.
|
|
13
13
|
# unreachable: a Method object captured before re-evaluation still calls
|
|
14
14
|
# the original body, so no test can observe the mutant. Outside the score.
|
|
15
|
-
|
|
15
|
+
# unverified: the --max-duration budget ran out before the verdict check
|
|
16
|
+
# or timeout calibration could run, so the kill or timeout proves
|
|
17
|
+
# nothing. Outside the score; the run reports budget-partial.
|
|
18
|
+
STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable unverified].freeze
|
|
16
19
|
|
|
17
20
|
# Outcome of one mutation. reason carries the skip reason, the error
|
|
18
21
|
# message or the timeout limit. ignored is the ignore-list reason, if any.
|
|
@@ -20,6 +23,11 @@ module Saboteur
|
|
|
20
23
|
attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
|
|
21
24
|
:kill_cause, :executed
|
|
22
25
|
attr_accessor :ignored
|
|
26
|
+
# The ignore list's triage for this survivor (#155): a state from
|
|
27
|
+
# IgnoreList::STATES and its reason. accepted_equivalent and wont_fix
|
|
28
|
+
# suppress the exit failure (the survivor still counts in the score); a
|
|
29
|
+
# todo survivor keeps counting as an unignored survivor.
|
|
30
|
+
attr_accessor :triage_state, :triage_reason
|
|
23
31
|
|
|
24
32
|
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
25
33
|
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
|
|
@@ -38,6 +46,17 @@ module Saboteur
|
|
|
38
46
|
@rejected_kill = rejected_kill
|
|
39
47
|
@reused = reused
|
|
40
48
|
@ignored = nil
|
|
49
|
+
# triage_state/triage_reason stay unset until the ignore list writes
|
|
50
|
+
# them: the accessors read as nil before that, and there is no
|
|
51
|
+
# statement for a mutation to remove (#304 dogfood).
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The same mutant, marked unverified: the budget ran out before its
|
|
55
|
+
# verdict could be checked, so it must not count as detected (#296).
|
|
56
|
+
def as_unverified(reason)
|
|
57
|
+
MutantResult.new(subject: subject, mutation: mutation, status: :unverified, mutated_source: mutated_source,
|
|
58
|
+
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
|
|
59
|
+
kill_cause: kill_cause, executed: executed)
|
|
41
60
|
end
|
|
42
61
|
|
|
43
62
|
# The same mutant, marked as an error: its verdict did not hold up.
|
|
@@ -100,10 +119,17 @@ module Saboteur
|
|
|
100
119
|
end
|
|
101
120
|
end
|
|
102
121
|
|
|
103
|
-
# A run that stopped before every selected subject was mutated (--fail-fast
|
|
104
|
-
#
|
|
105
|
-
# the
|
|
106
|
-
|
|
122
|
+
# A run that stopped before every selected subject was mutated (--fail-fast,
|
|
123
|
+
# --max-duration), or whose budget ran out before every verdict check.
|
|
124
|
+
# unreached lists the subject ids that never ran; truncated lists the
|
|
125
|
+
# subjects that started but stopped mid-way as {subject:, omitted:}
|
|
126
|
+
# entries; unverified lists the subjects whose kills the budget left
|
|
127
|
+
# unchecked. The verdict covers only completed, verified work.
|
|
128
|
+
EarlyStop = Data.define(:reason, :unreached, :truncated, :unverified, :planned_mutants) do
|
|
129
|
+
def initialize(reason:, unreached:, truncated:, unverified:, planned_mutants: nil)
|
|
130
|
+
super
|
|
131
|
+
end
|
|
132
|
+
end
|
|
107
133
|
|
|
108
134
|
# Everything one run produced.
|
|
109
135
|
class Report
|
|
@@ -113,14 +139,14 @@ module Saboteur
|
|
|
113
139
|
|
|
114
140
|
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
|
|
115
141
|
:generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions,
|
|
116
|
-
:operator_policy
|
|
142
|
+
:operator_policy, :discovery_failures, :memory, :shard
|
|
117
143
|
|
|
118
144
|
# REVIEW: diff mode; the terminal then shows one survivor per line.
|
|
119
145
|
# seed: the random seed every test run started from (nil when unknown).
|
|
120
146
|
def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
|
|
121
147
|
seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
|
|
122
148
|
run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [],
|
|
123
|
-
operator_policy: nil)
|
|
149
|
+
operator_policy: nil, discovery_failures: [], memory: nil, shard: nil)
|
|
124
150
|
@results = results
|
|
125
151
|
@subjects = subjects
|
|
126
152
|
@wall_time = wall_time
|
|
@@ -139,6 +165,11 @@ module Saboteur
|
|
|
139
165
|
@matrix = matrix
|
|
140
166
|
@inline_suppressions = inline_suppressions.dup.freeze
|
|
141
167
|
@operator_policy = operator_policy
|
|
168
|
+
@discovery_failures = discovery_failures.dup.freeze
|
|
169
|
+
# Per-worker and supervisor peak RSS in MB, when the platform measured
|
|
170
|
+
# it (#202); nil elsewhere, so JSON reports omit the section.
|
|
171
|
+
@memory = memory.dup.freeze
|
|
172
|
+
@shard = shard
|
|
142
173
|
end
|
|
143
174
|
|
|
144
175
|
# A --matrix run: every covering test ran per mutant.
|