saboteur 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/lib/saboteur/cli/output_options.rb +4 -0
- data/lib/saboteur/cli/run_command.rb +7 -1
- data/lib/saboteur/cli/run_reporting.rb +1 -1
- data/lib/saboteur/cli/survivors_command.rb +1 -0
- data/lib/saboteur/cli.rb +1 -1
- data/lib/saboteur/config.rb +19 -2
- data/lib/saboteur/engine/diagnostics.rb +4 -3
- data/lib/saboteur/engine.rb +35 -7
- data/lib/saboteur/reporter/default_findings.rb +83 -0
- data/lib/saboteur/reporter/next_steps.rb +1 -0
- data/lib/saboteur/reporter/review_comments.rb +3 -1
- data/lib/saboteur/reporter/speed_footer.rb +74 -0
- data/lib/saboteur/reporter/survivor_classification.rb +8 -3
- data/lib/saboteur/reporter/survivor_hints.rb +2 -1
- data/lib/saboteur/reporter/survivor_list.rb +27 -6
- data/lib/saboteur/reporter/terminal.rb +28 -10
- data/lib/saboteur/result.rb +18 -2
- data/lib/saboteur/survivor_guidance.rb +2 -2
- data/lib/saboteur/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7e61c3755190680c12faf62b3a0bcad5f74750da7ff3e3716736e8b9aa733c46
|
|
4
|
+
data.tar.gz: 85dc6bbddcc174034da44607b55c5c438c883bc5e0c5954a68d8115abb4b5984
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eceece31e29ceb03758b67aaa4ecfddcf1f6ca1f864358706fabcb8c9fed1f725da5548cd4f810684ac85250651f234b182452dd0525309711b616b997bf8026
|
|
7
|
+
data.tar.gz: f62d86ac983b24a550ba5696372fee7f9883be0103e823997f5fdd64c7150b42a8d88aae38881cd65ecd3b2141b39c22c9b7513552506a6499dddceb30128a69
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,35 @@ All notable changes are documented here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.12.0] - 2026-10-22
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- The default report is short: a findings list, the top three expanded
|
|
14
|
+
survivor blocks and `--details` for the rest, with a per-phase
|
|
15
|
+
wall-time speed footer and the slowest mutant. Fixed template lines
|
|
16
|
+
fit 80 columns and the piped-progress margin is wider so the summary
|
|
17
|
+
fits 80-column logs (#1256, #1152).
|
|
18
|
+
- A run on a Rails app with no `config/saboteur.rb` now defaults
|
|
19
|
+
`include` to `app/**/*.rb` and says so in the missing-config notice,
|
|
20
|
+
instead of exiting 2. A written config, even an empty one, keeps the
|
|
21
|
+
plain defaults, and a CLI `--include` still wins (#1257, #1171).
|
|
22
|
+
- The terminal summary prints one dim line under Score when mutants sit
|
|
23
|
+
on lines no test runs, saying they are not in the score and that the
|
|
24
|
+
JSON report lists them as `NoCoverage` (#1259, #1190).
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- Uncovered survivors are labeled `input_gap`, separate from assertion
|
|
29
|
+
gaps (#1254, #1157).
|
|
30
|
+
|
|
31
|
+
### Tests
|
|
32
|
+
|
|
33
|
+
- End-to-end pins for `--shard K/N` and the score history, badge and
|
|
34
|
+
trend commands, and for a stub-only method reported as not executed
|
|
35
|
+
(#1260, #1261, #1158, #1159). The survivor-ranking regression
|
|
36
|
+
fixture is the real money gem `allocation.rb` (#1262, #1160).
|
|
37
|
+
|
|
9
38
|
## [0.11.2] - 2026-10-22
|
|
10
39
|
|
|
11
40
|
### Added
|
|
@@ -42,6 +42,10 @@ module Saboteur
|
|
|
42
42
|
end
|
|
43
43
|
opts.on("--all-mutants", "Text report: list every survived mutant;",
|
|
44
44
|
"default: one finding per statement, with the label mix") { set(:all_mutants, true) }
|
|
45
|
+
# D1: the default whole-project report lists findings and expands the
|
|
46
|
+
# top three; --details restores a block for every finding.
|
|
47
|
+
opts.on("--details", "Text report: full blocks for every finding;",
|
|
48
|
+
"default: one-line list plus the top 3 blocks") { set(:details, true) }
|
|
45
49
|
opts.on("--no-color", "Deprecated alias for --color=never") { @color = :never }
|
|
46
50
|
end
|
|
47
51
|
end
|
|
@@ -25,7 +25,13 @@ module Saboteur
|
|
|
25
25
|
return if @format == "agent" || config.config_file_present
|
|
26
26
|
|
|
27
27
|
path = options[:config] || Config::FILE
|
|
28
|
-
notice =
|
|
28
|
+
notice =
|
|
29
|
+
if config["include"] == Config::RAILS_INCLUDE && !config.explicit?("include")
|
|
30
|
+
"saboteur: no #{path} - Rails app detected, mutating app/**/*.rb " \
|
|
31
|
+
"(bundle exec rails generate saboteur:install writes a config)"
|
|
32
|
+
else
|
|
33
|
+
"saboteur: no #{path} - using defaults (Rails: bundle exec rails generate saboteur:install writes one)"
|
|
34
|
+
end
|
|
29
35
|
@err.puts(palette.paint(:dim, notice))
|
|
30
36
|
end
|
|
31
37
|
|
|
@@ -21,7 +21,7 @@ module Saboteur
|
|
|
21
21
|
Reporter::Terminal.new(@out, verbose: @verbose || false, quiet: @quiet || false, color: @color || :auto,
|
|
22
22
|
overrides: config["palette"], threshold: config["threshold"],
|
|
23
23
|
replay_command: survivor_replay_command(report, engine),
|
|
24
|
-
all_mutants: options[:all_mutants] || false,
|
|
24
|
+
all_mutants: options[:all_mutants] || false, details: options[:details] || false,
|
|
25
25
|
exit_code: code, previous_score: previous_score, scope: scope_label,
|
|
26
26
|
whole_suite_map: whole_suite_map_for(engine),
|
|
27
27
|
run_map: engine&.run_coverage_map,
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -139,7 +139,7 @@ module Saboteur
|
|
|
139
139
|
|
|
140
140
|
def agent_requested? = self.class.agent_requested?(@argv)
|
|
141
141
|
|
|
142
|
-
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run, :all_mutants, :quiet)
|
|
142
|
+
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run, :all_mutants, :quiet, :details)
|
|
143
143
|
|
|
144
144
|
def add_subject(expr) = (options[:subjects] ||= []) << expr
|
|
145
145
|
|
data/lib/saboteur/config.rb
CHANGED
|
@@ -18,6 +18,11 @@ module Saboteur
|
|
|
18
18
|
LEGACY_IGNORE_FILE = ".saboteur-ignore.yml"
|
|
19
19
|
IGNORE_FILE = "config/saboteur_ignore.rb"
|
|
20
20
|
|
|
21
|
+
# A Rails app (config/application.rb + app/) without a config file gets
|
|
22
|
+
# the generator's include instead of the gem-shaped default, so a first
|
|
23
|
+
# run mutates app/ rather than refusing over an empty source set (#1171).
|
|
24
|
+
RAILS_INCLUDE = ["app/**/*.rb"].freeze
|
|
25
|
+
|
|
21
26
|
# Settings that say where saboteur writes, never what it mutates: left
|
|
22
27
|
# out of the run fingerprint, baseline evidence and score-history scope.
|
|
23
28
|
OUTPUT_KEYS = %w[cache_dir json_report html_report score_history resume].freeze
|
|
@@ -145,9 +150,21 @@ module Saboteur
|
|
|
145
150
|
"saboteur no longer reads #{LEGACY_IGNORE_FILE}; move its entries to #{IGNORE_FILE} (see README)."
|
|
146
151
|
end
|
|
147
152
|
|
|
148
|
-
|
|
149
|
-
|
|
153
|
+
file_present = File.file?(path)
|
|
154
|
+
from_file = file_present ? ConfigFile.load(path) : {}
|
|
155
|
+
# Only a missing file triggers detection: a written config, even an
|
|
156
|
+
# empty one, means the defaults were chosen. The detected include is
|
|
157
|
+
# not an explicit key - the user set nothing.
|
|
158
|
+
detected = !file_present && rails_app?(root) ? { "include" => RAILS_INCLUDE } : {}
|
|
159
|
+
cli = stringify(overrides)
|
|
160
|
+
merged = detected.merge(from_file).merge(cli)
|
|
161
|
+
new(merged, root: root, explicit: from_file.keys | cli.keys, config_file_present: file_present)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def self.rails_app?(root)
|
|
165
|
+
File.file?(File.join(root, "config", "application.rb")) && File.directory?(File.join(root, "app"))
|
|
150
166
|
end
|
|
167
|
+
private_class_method :rails_app?
|
|
151
168
|
|
|
152
169
|
def self.stringify(hash)
|
|
153
170
|
hash.to_h { |key, value| [key.to_s, value] }.compact
|
|
@@ -77,9 +77,10 @@ module Saboteur
|
|
|
77
77
|
# progress (see start_progress); the letter is legend-only.
|
|
78
78
|
LEGEND = ".:killed S:survived T:timeout E:error s:skipped L:resource limit"
|
|
79
79
|
|
|
80
|
-
# Chunk sizing: leave room for the "
|
|
81
|
-
# below the floor
|
|
82
|
-
|
|
80
|
+
# Chunk sizing: leave room for the full " n/total (100%) ETA 123m45s"
|
|
81
|
+
# summary suffix (32 columns), never below the floor, so a piped line
|
|
82
|
+
# fits an 80-column CI log instead of wrapping mid-summary.
|
|
83
|
+
CHUNK_MARGIN = 32
|
|
83
84
|
CHUNK_FLOOR = 40
|
|
84
85
|
DEFAULT_WIDTH = 80
|
|
85
86
|
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -199,7 +199,8 @@ module Saboteur
|
|
|
199
199
|
matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
|
|
200
200
|
operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures,
|
|
201
201
|
test_command: config["test_command"], reuse_label: reuse_label, since: config["since"],
|
|
202
|
-
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {}
|
|
202
|
+
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {},
|
|
203
|
+
phase_timings: @phase_timings)
|
|
203
204
|
end
|
|
204
205
|
|
|
205
206
|
def complete_evidence(results, subjects)
|
|
@@ -226,14 +227,13 @@ module Saboteur
|
|
|
226
227
|
|
|
227
228
|
def mutate(subjects, files, integration, scope, ignore_list, load_lines: {}, load_methods: {})
|
|
228
229
|
store = coverage_store(integration, load_lines, load_methods)
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
check_baseline(integration, full_passes)
|
|
230
|
+
map_reusable = store.reusable?(files)
|
|
231
|
+
full_passes = config["baseline_passes"] + (map_reusable ? 0 : 1)
|
|
232
|
+
baseline_preflight(store, integration, files, full_passes)
|
|
233
233
|
refuse_unaffordable_coverage_build(store, files)
|
|
234
234
|
return [] if abandon_for_budget?(subjects)
|
|
235
235
|
|
|
236
|
-
map = store
|
|
236
|
+
map, map_seconds = timed_map_fetch(store, files)
|
|
237
237
|
inform_about_run(subjects, map)
|
|
238
238
|
cache = result_cache(map, files)
|
|
239
239
|
scheduler = scheduler_for(map, integration, cache)
|
|
@@ -246,13 +246,41 @@ module Saboteur
|
|
|
246
246
|
# mutant when serial. Pack the heap now that the map and candidates
|
|
247
247
|
# are retained, to reduce later copying of shared pages.
|
|
248
248
|
Execution::ForkReadiness.prepare_heap
|
|
249
|
-
results =
|
|
249
|
+
results = run_and_confirm_timed(runnable_subjects, runnable_lists, scheduler, cache, ignore_list,
|
|
250
|
+
map_cached: map_reusable, map_seconds:)
|
|
250
251
|
cache&.save
|
|
251
252
|
results
|
|
252
253
|
rescue BudgetPhaseTimeout => e
|
|
253
254
|
budget_phase_stop(e, subjects)
|
|
254
255
|
end
|
|
255
256
|
|
|
257
|
+
# The map phase's monotonic split, for the speed footer.
|
|
258
|
+
def timed_map_fetch(store, files)
|
|
259
|
+
started = monotonic
|
|
260
|
+
[store.fetch(files), monotonic - started]
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# Baseline phase: announce, then verify the unchanged tree passes.
|
|
264
|
+
def baseline_preflight(store, integration, files, full_passes)
|
|
265
|
+
warn_unscoped(integration, full_passes)
|
|
266
|
+
announce_quiet_start(store, files)
|
|
267
|
+
check_baseline(integration, full_passes)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# The mutant phase plus the run's three disjoint timing intervals
|
|
271
|
+
# (baseline inside check_baseline, map build, mutants) and the worker
|
|
272
|
+
# count, recorded for the terminal's speed footer.
|
|
273
|
+
def run_and_confirm_timed(subjects, lists, scheduler, cache, ignore_list, map_cached:, map_seconds:)
|
|
274
|
+
started = monotonic
|
|
275
|
+
results = run_and_confirm(subjects, lists, scheduler, cache, ignore_list)
|
|
276
|
+
# @baseline_elapsed is a Float from initialize on, so no fallback is needed.
|
|
277
|
+
@phase_timings = PhaseTimings.new(baseline_seconds: @baseline_elapsed,
|
|
278
|
+
map_seconds:, map_cached:,
|
|
279
|
+
mutants_seconds: monotonic - started,
|
|
280
|
+
jobs: config["jobs"])
|
|
281
|
+
results
|
|
282
|
+
end
|
|
283
|
+
|
|
256
284
|
# The candidate run plus the opt-in post-run confirmation pass (#1063):
|
|
257
285
|
# confirmation runs after the last result, before the verdict cache is
|
|
258
286
|
# written, so a confirmed kill overwrites the survivor verdict the run
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "mutation_text"
|
|
5
|
+
require_relative "survivor_classification"
|
|
6
|
+
require_relative "survivor_list"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module Reporter
|
|
10
|
+
# Default whole-project findings (D1): the one-line-per-finding list the
|
|
11
|
+
# --quiet report popularised, then full blocks for the top
|
|
12
|
+
# SurvivorRanking picks only, then a dim closer naming the rest. A
|
|
13
|
+
# --since run or --details keeps the full block list.
|
|
14
|
+
class DefaultFindings
|
|
15
|
+
# Blocks a default run expands; the closer counts what is left out.
|
|
16
|
+
TOP_BLOCKS = 3
|
|
17
|
+
|
|
18
|
+
# Two lines: the fixed text alone is 87 columns at rest; keep it inside 80.
|
|
19
|
+
CLOSER = "... %<more>d more %<findings>s - saboteur survivors .saboteur/report.json,\nor re-run with --details"
|
|
20
|
+
|
|
21
|
+
def initialize(out, palette, all_mutants: false, details: false,
|
|
22
|
+
whole_suite_map: SurvivorList::NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
23
|
+
@out = out
|
|
24
|
+
@palette = palette
|
|
25
|
+
@all_mutants = all_mutants
|
|
26
|
+
@details = details
|
|
27
|
+
@whole_suite_map = whole_suite_map
|
|
28
|
+
@run_map = run_map
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def call(report, ranked)
|
|
32
|
+
live = ranked.select(&:unignored_survivor?)
|
|
33
|
+
return full(report, ranked) if @details || report.diff? || live.empty?
|
|
34
|
+
|
|
35
|
+
@out.puts(list_lines(report, live))
|
|
36
|
+
@out.puts
|
|
37
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants, limit: TOP_BLOCKS,
|
|
38
|
+
whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
|
|
39
|
+
more = finding_count(report, live) - TOP_BLOCKS
|
|
40
|
+
findings = "finding#{'s' unless more == 1}"
|
|
41
|
+
@out.puts @palette.paint(:dim, format(CLOSER, more:, findings:)) if more.positive?
|
|
42
|
+
@out.puts
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# --details and --since runs: every finding expanded, exactly the
|
|
48
|
+
# pre-D1 default.
|
|
49
|
+
def full(report, ranked)
|
|
50
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants,
|
|
51
|
+
whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
|
|
52
|
+
@out.puts unless report.survivors.empty?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The quiet list's line shape: path:line subject, label and digest,
|
|
56
|
+
# sibling mutants on the statement collapsed to a count (#1126/#1133).
|
|
57
|
+
def list_lines(report, live)
|
|
58
|
+
groups(report, live).map do |(path, line, subject_id), results|
|
|
59
|
+
first, *rest = results
|
|
60
|
+
label = SurvivorClassification.label(first)
|
|
61
|
+
text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
|
|
62
|
+
rest.any? ? "#{text} (+#{rest.size} more)" : text
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# --all-mutants gives every survivor its own block, so the closer
|
|
67
|
+
# counts mutants rather than statements.
|
|
68
|
+
def finding_count(report, live)
|
|
69
|
+
@all_mutants ? live.size : groups(report, live).size
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def groups(report, live)
|
|
73
|
+
live.group_by do |result|
|
|
74
|
+
[relative(result.subject.file, report.root), MutationText.line(result), result.subject.id]
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def relative(file, root)
|
|
79
|
+
file.delete_prefix("#{root}/")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -33,6 +33,7 @@ module Saboteur
|
|
|
33
33
|
@out.puts " #{SurvivorGuidance::REPLAY_CAVEAT}"
|
|
34
34
|
end
|
|
35
35
|
@out.puts " #{SurvivorGuidance::IGNORE_INTRO}"
|
|
36
|
+
@out.puts " #{SurvivorGuidance::IGNORE_CONFIG}"
|
|
36
37
|
@out.puts " Saboteur.ignore do |i|"
|
|
37
38
|
@out.puts " #{SurvivorGuidance.ignore_entry(survivor.digest, survivor.subject.id)}"
|
|
38
39
|
@out.puts " end"
|
|
@@ -30,7 +30,9 @@ module Saboteur
|
|
|
30
30
|
GAP_HINT = "A test reaches this line and still passes with the change. " \
|
|
31
31
|
"Add an assertion that pins this behaviour."
|
|
32
32
|
TOOL_LIMIT_HINT = "Saboteur could not observe this mutant faithfully. Check it by hand."
|
|
33
|
-
HINTS = GapKinds::KINDS.to_h { |label| [label, GAP_HINT] }
|
|
33
|
+
HINTS = GapKinds::KINDS.to_h { |label| [label, GAP_HINT] }
|
|
34
|
+
.merge("tool_limit" => TOOL_LIMIT_HINT,
|
|
35
|
+
"input_gap" => "No test reaches this line. Exercise it with a varied input.").freeze
|
|
34
36
|
DEFAULT_HINT = "No test fails with this change. Check whether a test should pin it."
|
|
35
37
|
|
|
36
38
|
Plan = Struct.new(:post, :delete, :summary, keyword_init: true)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# PR16's speed footer: the run's wall time split by phase, so
|
|
7
|
+
# "saboteur is slow" gets a number with an owner (Stryker and
|
|
8
|
+
# cargo-mutants both print phase timings). A diagnostic, not part of
|
|
9
|
+
# the clean screen: default runs print it only when an overhead phase
|
|
10
|
+
# (baseline or map - never the mutant phase, which IS the work) ate
|
|
11
|
+
# more than OVERHEAD_SHARE of the wall past OVERHEAD_FLOOR seconds;
|
|
12
|
+
# --verbose always prints it.
|
|
13
|
+
class SpeedFooter
|
|
14
|
+
OVERHEAD_SHARE = 0.25
|
|
15
|
+
OVERHEAD_FLOOR = 2.0
|
|
16
|
+
# A mutant slower than SLOW_MUTANT_FACTOR times the median mutant
|
|
17
|
+
# past SLOW_MUTANT_FLOOR seconds is an outlier worth naming.
|
|
18
|
+
SLOW_MUTANT_FACTOR = 5
|
|
19
|
+
SLOW_MUTANT_FLOOR = 1.0
|
|
20
|
+
|
|
21
|
+
def initialize(out, palette, verbose: false)
|
|
22
|
+
@out = out
|
|
23
|
+
@palette = palette
|
|
24
|
+
@verbose = verbose
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call(report)
|
|
28
|
+
timings = report.phase_timings
|
|
29
|
+
return unless timings
|
|
30
|
+
return unless @verbose || overhead_exceeded?(timings, report.wall_time)
|
|
31
|
+
|
|
32
|
+
@out.puts @palette.paint(:dim, line(timings, report.wall_time))
|
|
33
|
+
outlier = slowest_outlier(report.results)
|
|
34
|
+
return unless outlier
|
|
35
|
+
|
|
36
|
+
@out.puts @palette.paint(:dim, " slowest: #{outlier.subject.id} " \
|
|
37
|
+
"(#{outlier.mutation.operator}, #{format('%.1f', outlier.runtime)}s)")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def overhead_exceeded?(timings, wall_time)
|
|
43
|
+
[timings.baseline_seconds, timings.map_seconds].any? do |seconds|
|
|
44
|
+
seconds > OVERHEAD_FLOOR && seconds > wall_time * OVERHEAD_SHARE
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def line(timings, wall_time)
|
|
49
|
+
map = timings.map_cached ? "0.0s (cached)" : "#{format('%.1f', timings.map_seconds)}s"
|
|
50
|
+
"speed footer: done in #{format('%.1f', wall_time)}s - " \
|
|
51
|
+
"baseline #{format('%.1f', timings.baseline_seconds)}s #{middot} map #{map} #{middot} " \
|
|
52
|
+
"mutants #{format('%.1f', timings.mutants_seconds)}s (jobs=#{timings.jobs})"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def slowest_outlier(results)
|
|
56
|
+
timed = results.select(&:runtime)
|
|
57
|
+
return if timed.empty?
|
|
58
|
+
|
|
59
|
+
slowest = timed.max_by(&:runtime)
|
|
60
|
+
return unless slowest.runtime > SLOW_MUTANT_FLOOR && slowest.runtime > median(timed) * SLOW_MUTANT_FACTOR
|
|
61
|
+
|
|
62
|
+
slowest
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def median(results)
|
|
66
|
+
runtimes = results.map(&:runtime).sort
|
|
67
|
+
middle = runtimes.size / 2
|
|
68
|
+
runtimes.size.odd? ? runtimes[middle] : (runtimes[middle - 1] + runtimes[middle]) / 2.0
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def middot = @palette.glyph(:middot)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -28,8 +28,12 @@ module Saboteur
|
|
|
28
28
|
# behaviour; the kind names WHAT sort of check is missing, from the
|
|
29
29
|
# operator and the edit shape only. The covering list is only the
|
|
30
30
|
# PLANNED coverage: whole-body (coarse) mutants carry method-level
|
|
31
|
-
# guesses, so they never get a gap kind
|
|
32
|
-
#
|
|
31
|
+
# guesses, so they never get a gap kind.
|
|
32
|
+
# input_gap - no real test covered the mutant at all, so the input
|
|
33
|
+
# never varied (D6). Coarse mutants stay unlabeled: their planned
|
|
34
|
+
# coverage is a method-level guess, not an observation. The
|
|
35
|
+
# deterministic version (the runner records the mutated value
|
|
36
|
+
# actually differing) is #1064 follow-up.
|
|
33
37
|
# tool_limit - survivors saboteur could not observe faithfully: an
|
|
34
38
|
# external-runner run records only the pseudo-test id, never real
|
|
35
39
|
# coverage (#1071). A re-eval refusal records no signal on the
|
|
@@ -37,7 +41,7 @@ module Saboteur
|
|
|
37
41
|
#
|
|
38
42
|
# A survivor with no matching signal stays unlabeled.
|
|
39
43
|
module SurvivorClassification
|
|
40
|
-
LABELS = %w[likely_equivalent arid tool_limit].concat(GapKinds::KINDS).freeze
|
|
44
|
+
LABELS = %w[likely_equivalent arid tool_limit input_gap].concat(GapKinds::KINDS).freeze
|
|
41
45
|
|
|
42
46
|
class << self
|
|
43
47
|
def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
|
|
@@ -46,6 +50,7 @@ module Saboteur
|
|
|
46
50
|
return "arid" if arid?(result, arid)
|
|
47
51
|
return "tool_limit" if tool_limited?(result)
|
|
48
52
|
return gap_kind(result) if possible_test_gap?(result)
|
|
53
|
+
return "input_gap" unless result.mutation.coarse
|
|
49
54
|
|
|
50
55
|
nil
|
|
51
56
|
end
|
|
@@ -96,7 +96,8 @@ module Saboteur
|
|
|
96
96
|
LABEL_REASONS = {
|
|
97
97
|
"likely_equivalent" => "compiles to the same instructions as the original",
|
|
98
98
|
"arid" => "every edited line matches an arid pattern",
|
|
99
|
-
"tool_limit" => "the tool could not observe this mutant faithfully"
|
|
99
|
+
"tool_limit" => "the tool could not observe this mutant faithfully",
|
|
100
|
+
"input_gap" => "no test reached this mutant, so its input never varied"
|
|
100
101
|
}.merge(GapKinds::REASONS).freeze
|
|
101
102
|
|
|
102
103
|
module_function
|
|
@@ -34,26 +34,31 @@ module Saboteur
|
|
|
34
34
|
SIBLING_DIGEST_CAP = 3
|
|
35
35
|
|
|
36
36
|
def initialize(out, palette, all_mutants: false, classifier: SurvivorClassification,
|
|
37
|
-
whole_suite_map: NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
37
|
+
whole_suite_map: NO_WHOLE_SUITE_LOOKUP, run_map: nil, limit: nil)
|
|
38
38
|
@out = out
|
|
39
39
|
@palette = palette
|
|
40
40
|
@all_mutants = all_mutants
|
|
41
41
|
@classifier = classifier
|
|
42
42
|
@whole_suite_map = whole_suite_map
|
|
43
43
|
@run_map = run_map
|
|
44
|
+
@limit = limit
|
|
44
45
|
end
|
|
45
46
|
|
|
46
47
|
# Survivors in report (ranked) order; accepted ones trail as dim lines.
|
|
48
|
+
# A limit renders only the first groups as blocks - the caller names
|
|
49
|
+
# the rest itself - while the header still counts every live survivor.
|
|
47
50
|
def call(report, survivors)
|
|
48
51
|
return if survivors.empty?
|
|
49
52
|
|
|
50
53
|
live, accepted = survivors.partition(&:unignored_survivor?)
|
|
51
54
|
@out.puts @palette.paint(:head, "Survivors (#{live.size}) - changes your tests did not catch")
|
|
52
55
|
block_list = groups(live)
|
|
53
|
-
|
|
56
|
+
total = block_list.size
|
|
57
|
+
shown = @limit ? block_list.first(@limit) : block_list
|
|
58
|
+
shown.each_with_index { |group, index| block(report, group, index + 1, total) }
|
|
54
59
|
accepted_section(report, accepted)
|
|
55
|
-
hinted = live.select { |result| hint_for(result) }
|
|
56
|
-
@out.puts "\n#{@palette.paint(:dim, SurvivorHints::CAVEAT)}" if hinted.any?
|
|
60
|
+
hinted = @limit ? shown.flatten.select { |result| hint_for(result) } : live.select { |result| hint_for(result) }
|
|
61
|
+
@out.puts "\n#{@palette.paint(:dim, wrap(SurvivorHints::CAVEAT))}" if hinted.any?
|
|
57
62
|
end
|
|
58
63
|
|
|
59
64
|
private
|
|
@@ -73,6 +78,8 @@ module Saboteur
|
|
|
73
78
|
path = relative(first.subject.file, report.root)
|
|
74
79
|
@out.puts "\n#{@palette.paint(:head, "#{rank}. #{path}:#{line}")} #{first.subject.id} " \
|
|
75
80
|
"#{@palette.paint(:dim, label_suffix(first, rank, total))}".rstrip
|
|
81
|
+
reason = SurvivorHints.label_reason(@classifier.label(first))
|
|
82
|
+
@out.puts " #{@palette.paint(:dim, reason)}" if reason
|
|
76
83
|
fence(first, line)
|
|
77
84
|
@out.puts " #{@palette.paint(:dim, "why it matters: #{why(first.mutation.operator)}")}"
|
|
78
85
|
changed_lines(report, path, first)
|
|
@@ -81,14 +88,28 @@ module Saboteur
|
|
|
81
88
|
siblings(report, group)
|
|
82
89
|
end
|
|
83
90
|
|
|
91
|
+
# Fixed furniture fits an 80-column log: word-wrap the caveat. The
|
|
92
|
+
# JSON agent format keeps SurvivorHints::CAVEAT as one line; only the
|
|
93
|
+
# terminal rendering wraps. Interpolated lines are exempt.
|
|
94
|
+
def wrap(text, width = 80)
|
|
95
|
+
text.split.each_with_object([+""]) do |word, lines|
|
|
96
|
+
if !lines.last.empty? && lines.last.length + word.length + 1 > width
|
|
97
|
+
lines << +word
|
|
98
|
+
else
|
|
99
|
+
lines.last << (lines.last.empty? ? word : " #{word}")
|
|
100
|
+
end
|
|
101
|
+
end.join("\n")
|
|
102
|
+
end
|
|
103
|
+
|
|
84
104
|
# The count names blocks, the unit the leading ordinal uses: grouped
|
|
85
105
|
# siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
|
|
106
|
+
# Label and rank stay on the header line; the reason moves to its own
|
|
107
|
+
# dim line so the fixed text cannot push the header past 80 columns.
|
|
86
108
|
def label_suffix(result, rank, total)
|
|
87
109
|
label = @classifier.label(result)
|
|
88
110
|
unit = total == 1 ? "block" : "blocks"
|
|
89
111
|
parts = [("coarse" if result.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
|
|
90
|
-
|
|
91
|
-
"[#{parts.join(" \u00B7 ")}#{": #{reason}" if reason}]"
|
|
112
|
+
"[#{parts.join(' · ')}]"
|
|
92
113
|
end
|
|
93
114
|
|
|
94
115
|
# The enclosing def with the changed line as a -/+ pair; longer
|
|
@@ -10,6 +10,8 @@ require_relative "survivor_ranking"
|
|
|
10
10
|
require_relative "palette"
|
|
11
11
|
require_relative "notice"
|
|
12
12
|
require_relative "score_table"
|
|
13
|
+
require_relative "speed_footer"
|
|
14
|
+
require_relative "default_findings"
|
|
13
15
|
require_relative "survivor_list"
|
|
14
16
|
require_relative "survivor_classification"
|
|
15
17
|
require_relative "test_matrix"
|
|
@@ -26,12 +28,13 @@ module Saboteur
|
|
|
26
28
|
MAX_PASSING_TESTS = 5
|
|
27
29
|
|
|
28
30
|
def initialize(out, verbose: false, quiet: false, color: :auto, overrides: nil,
|
|
29
|
-
replay_command: nil, all_mutants: false, exit_code: nil,
|
|
31
|
+
replay_command: nil, all_mutants: false, details: false, exit_code: nil,
|
|
30
32
|
threshold: nil, previous_score: nil, baseline_gate: nil,
|
|
31
33
|
scope: "whole project", baseline: nil, coverage: nil, mutant_phase: nil,
|
|
32
34
|
whole_suite_map: SurvivorList::NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
33
35
|
@out = SafeOut.new(out)
|
|
34
36
|
@all_mutants = all_mutants
|
|
37
|
+
@details = details
|
|
35
38
|
@verbose = verbose
|
|
36
39
|
@quiet = quiet
|
|
37
40
|
# Clean-screen context: scope from the CLI; PR4a plumbs the phase stats.
|
|
@@ -58,7 +61,7 @@ module Saboteur
|
|
|
58
61
|
def call(report)
|
|
59
62
|
return quiet_call(report) if @quiet
|
|
60
63
|
|
|
61
|
-
|
|
64
|
+
findings(report)
|
|
62
65
|
errors(report)
|
|
63
66
|
skipped_subjects(report) if @verbose
|
|
64
67
|
test_command_gaps(report)
|
|
@@ -69,6 +72,7 @@ module Saboteur
|
|
|
69
72
|
test_matrix(report)
|
|
70
73
|
operator_policy(report)
|
|
71
74
|
summary(report)
|
|
75
|
+
SpeedFooter.new(@out, @palette, verbose: @verbose).call(report)
|
|
72
76
|
end
|
|
73
77
|
|
|
74
78
|
private
|
|
@@ -161,6 +165,7 @@ module Saboteur
|
|
|
161
165
|
@out.puts @palette.paint(:dim, tally)
|
|
162
166
|
@out.puts report.shard.line if report.shard
|
|
163
167
|
score_headline(report)
|
|
168
|
+
no_coverage_note(report)
|
|
164
169
|
details(report)
|
|
165
170
|
@score_table.call(report)
|
|
166
171
|
not_scored(report)
|
|
@@ -210,9 +215,7 @@ module Saboteur
|
|
|
210
215
|
@out.puts "#{dots} #{suffix}"
|
|
211
216
|
end
|
|
212
217
|
|
|
213
|
-
def win_tally(report)
|
|
214
|
-
"#{report.counts[:killed]}/#{report.executed.size} mutants killed"
|
|
215
|
-
end
|
|
218
|
+
def win_tally(report) = "#{report.counts[:killed]}/#{report.executed.size} mutants killed"
|
|
216
219
|
|
|
217
220
|
def hazard_suppressions(report)
|
|
218
221
|
return unless report.hazard_suppressions.positive?
|
|
@@ -254,6 +257,20 @@ module Saboteur
|
|
|
254
257
|
@out.puts line
|
|
255
258
|
end
|
|
256
259
|
|
|
260
|
+
# NoCoverage mutants ride in the mutant count but outside the score
|
|
261
|
+
# (#1190): say how many sit on lines no test runs, and how to list
|
|
262
|
+
# them. Stryker.status maps this skip-reason prefix to NoCoverage.
|
|
263
|
+
def no_coverage_note(report)
|
|
264
|
+
count = report.results.count do |result|
|
|
265
|
+
result.status == :skipped && result.reason&.start_with?("no coverage")
|
|
266
|
+
end
|
|
267
|
+
return if count.zero?
|
|
268
|
+
|
|
269
|
+
counted = Text.counted(count, "mutant")
|
|
270
|
+
note = "#{counted} on lines no test runs (not in the score): the JSON report lists them as NoCoverage"
|
|
271
|
+
@out.puts " #{@palette.paint(:dim, note)}"
|
|
272
|
+
end
|
|
273
|
+
|
|
257
274
|
# The share of tested mutants the run detected: "killed" alone would
|
|
258
275
|
# count timeouts too, so Wording names both when one is in the number.
|
|
259
276
|
def score_parenthetical(breakdown)
|
|
@@ -592,11 +609,12 @@ module Saboteur
|
|
|
592
609
|
@out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
|
|
593
610
|
end
|
|
594
611
|
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
612
|
+
# Default whole-project runs keep the first screen short (D1): the
|
|
613
|
+
# findings list plus the top blocks; --details or --since expands all.
|
|
614
|
+
def findings(report)
|
|
615
|
+
DefaultFindings.new(@out, @palette, all_mutants: @all_mutants, details: @details,
|
|
616
|
+
whole_suite_map: @whole_suite_map,
|
|
617
|
+
run_map: @run_map).call(report, ranked_survivors(report))
|
|
600
618
|
end
|
|
601
619
|
|
|
602
620
|
# Live survivors precede accepted ones; each group follows SurvivorRanking.
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -162,6 +162,18 @@ module Saboteur
|
|
|
162
162
|
end
|
|
163
163
|
end
|
|
164
164
|
|
|
165
|
+
# Per-phase wall time of one run: baseline check, coverage map build,
|
|
166
|
+
# mutant execution, and the worker count (PR16). Recorded from monotonic
|
|
167
|
+
# clocks at the phase boundaries; a reused map reports 0.0 s (cached), so
|
|
168
|
+
# the three phases never double count the cached build and their sum stays
|
|
169
|
+
# inside the wall time.
|
|
170
|
+
PhaseTimings = Data.define(:baseline_seconds, :map_seconds, :map_cached, :mutants_seconds, :jobs) do
|
|
171
|
+
def initialize(baseline_seconds:, map_seconds:, map_cached:, mutants_seconds:, jobs:)
|
|
172
|
+
super(baseline_seconds:, map_seconds: map_cached ? 0.0 : map_seconds, map_cached:,
|
|
173
|
+
mutants_seconds:, jobs:)
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
165
177
|
# Everything one run produced.
|
|
166
178
|
class Report
|
|
167
179
|
# More than half of the executed mutants errored: the run tested
|
|
@@ -171,7 +183,8 @@ module Saboteur
|
|
|
171
183
|
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :live_announced, :root, :seed,
|
|
172
184
|
:skipped_subjects, :since,
|
|
173
185
|
:generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions,
|
|
174
|
-
:operator_policy, :discovery_failures, :memory, :shard, :test_command, :referenced, :reuse_label
|
|
186
|
+
:operator_policy, :discovery_failures, :memory, :shard, :test_command, :referenced, :reuse_label,
|
|
187
|
+
:phase_timings
|
|
175
188
|
|
|
176
189
|
# REVIEW: diff mode; the terminal then shows one survivor per line.
|
|
177
190
|
# seed: the random seed every test run started from (nil when unknown).
|
|
@@ -180,7 +193,7 @@ module Saboteur
|
|
|
180
193
|
seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
|
|
181
194
|
run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [],
|
|
182
195
|
operator_policy: nil, discovery_failures: [], memory: nil, shard: nil,
|
|
183
|
-
test_command: nil, referenced: {}, reuse_label: nil, since: nil)
|
|
196
|
+
test_command: nil, referenced: {}, reuse_label: nil, since: nil, phase_timings: nil)
|
|
184
197
|
@results = results
|
|
185
198
|
@subjects = subjects
|
|
186
199
|
@wall_time = wall_time
|
|
@@ -213,6 +226,9 @@ module Saboteur
|
|
|
213
226
|
# or require instead of by name (zero-setup selection).
|
|
214
227
|
@referenced = referenced
|
|
215
228
|
@reuse_label = reuse_label
|
|
229
|
+
# nil for pipelines without the three phases (--test-command runs):
|
|
230
|
+
# the terminal prints no speed footer without a full split.
|
|
231
|
+
@phase_timings = phase_timings
|
|
216
232
|
end
|
|
217
233
|
|
|
218
234
|
# A --matrix run: every covering test ran per mutant.
|
|
@@ -8,8 +8,8 @@ module Saboteur
|
|
|
8
8
|
# surfaces never drift (#1167).
|
|
9
9
|
module SurvivorGuidance
|
|
10
10
|
REPLAY_CAVEAT = "(add this run's own --config, --since, --subject, --test and source options)"
|
|
11
|
-
IGNORE_INTRO = "accept an equivalent one: add to your ignore file
|
|
12
|
-
|
|
11
|
+
IGNORE_INTRO = "accept an equivalent one: add to your ignore file"
|
|
12
|
+
IGNORE_CONFIG = "(c.ignore_file, default #{Config::IGNORE_FILE}):".freeze
|
|
13
13
|
REASON_PLACEHOLDER = "why it is equivalent"
|
|
14
14
|
|
|
15
15
|
module_function
|
data/lib/saboteur/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: saboteur
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vamsi Pavan Mahesh Gunturu
|
|
@@ -225,6 +225,7 @@ files:
|
|
|
225
225
|
- lib/saboteur/reporter/assets/mutation_report.LICENSE
|
|
226
226
|
- lib/saboteur/reporter/assets/mutation_report.js
|
|
227
227
|
- lib/saboteur/reporter/baseline_tail.rb
|
|
228
|
+
- lib/saboteur/reporter/default_findings.rb
|
|
228
229
|
- lib/saboteur/reporter/diff.rb
|
|
229
230
|
- lib/saboteur/reporter/findings.rb
|
|
230
231
|
- lib/saboteur/reporter/gap_kinds.rb
|
|
@@ -244,6 +245,7 @@ files:
|
|
|
244
245
|
- lib/saboteur/reporter/safe_out.rb
|
|
245
246
|
- lib/saboteur/reporter/score_history.rb
|
|
246
247
|
- lib/saboteur/reporter/score_table.rb
|
|
248
|
+
- lib/saboteur/reporter/speed_footer.rb
|
|
247
249
|
- lib/saboteur/reporter/stryker.rb
|
|
248
250
|
- lib/saboteur/reporter/survivor_classification.rb
|
|
249
251
|
- lib/saboteur/reporter/survivor_hints.rb
|