saboteur 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +97 -0
- data/README.md +16 -3
- data/lib/saboteur/baseline/command.rb +6 -1
- data/lib/saboteur/baseline/comparison.rb +8 -0
- data/lib/saboteur/baseline/completeness.rb +14 -4
- data/lib/saboteur/baseline/evidence.rb +8 -1
- data/lib/saboteur/baseline/run_evidence.rb +10 -1
- data/lib/saboteur/baseline/workflow.rb +17 -5
- data/lib/saboteur/cli/color_option.rb +26 -0
- data/lib/saboteur/cli/doctor_command.rb +1 -0
- data/lib/saboteur/cli/exit_status.rb +12 -7
- data/lib/saboteur/cli/explain_command.rb +2 -1
- data/lib/saboteur/cli/failure_output.rb +13 -2
- data/lib/saboteur/cli/help_command.rb +5 -0
- data/lib/saboteur/cli/history_command.rb +2 -0
- data/lib/saboteur/cli/output_options.rb +1 -5
- data/lib/saboteur/cli/run_command.rb +1 -1
- data/lib/saboteur/cli/run_reporting.rb +2 -10
- data/lib/saboteur/cli/run_validation.rb +0 -1
- data/lib/saboteur/cli/selection_options.rb +3 -2
- data/lib/saboteur/cli/survivors_command.rb +81 -24
- data/lib/saboteur/cli/whole_suite_map_lookup.rb +33 -0
- data/lib/saboteur/cli.rb +6 -2
- data/lib/saboteur/engine/coverage_store.rb +7 -4
- data/lib/saboteur/engine/diagnostics.rb +27 -5
- data/lib/saboteur/engine/pipelines.rb +9 -2
- data/lib/saboteur/engine/survivor_confirmation.rb +75 -11
- data/lib/saboteur/engine/test_guess.rb +2 -1
- data/lib/saboteur/engine.rb +37 -18
- data/lib/saboteur/execution/runner.rb +3 -0
- data/lib/saboteur/gates.rb +1 -1
- data/lib/saboteur/help.rb +5 -5
- data/lib/saboteur/integration/rspec.rb +30 -0
- data/lib/saboteur/reporter/default_findings.rb +1 -1
- data/lib/saboteur/reporter/findings.rb +10 -1
- data/lib/saboteur/reporter/gap_kinds.rb +6 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/next_steps.rb +3 -1
- data/lib/saboteur/reporter/score_table.rb +6 -11
- data/lib/saboteur/reporter/stryker.rb +7 -2
- data/lib/saboteur/reporter/survivor_classification.rb +15 -0
- data/lib/saboteur/reporter/survivor_list.rb +6 -6
- data/lib/saboteur/reporter/terminal.rb +6 -3
- data/lib/saboteur/reporter/unconfirmed_section.rb +44 -0
- data/lib/saboteur/result.rb +39 -2
- data/lib/saboteur/selection/coverage_map.rb +11 -1
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +4 -0
- metadata +4 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ce718289117e022594f50dfb13bfaf3feacf03f99dc1a5b73c7c2463581d24f2
|
|
4
|
+
data.tar.gz: f99866dbeacb50607ac66ee9f15582b49b1596b318cba1598afdb7bac89a9133
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2ef5f5dc4d8d441d35558569bd5649bc4d7bbece07bde9172bd6d660a8d54e0274cc02b36f2968f4e129080cc3ccc3c5bdb3a091c0fbe55341a67d53bf967540
|
|
7
|
+
data.tar.gz: 9f6d1c79c63abc5d314f79f699bb9ccfa9de5bdb2a8c7d4240d311854774c4eb4c3c40950234f206bb62872008b5525181bd20b13325b080f680ebc4227ad6e5
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,103 @@ All notable changes are documented here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.12.2] - 2026-10-22
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- A scoped run (`--since`, `--subject`, or the default spec inference)
|
|
14
|
+
confirms its survivors against the whole-suite coverage map when one
|
|
15
|
+
exists. Each survivor gets one budgeted confirmation run of the covering
|
|
16
|
+
specs the run did not load. A mutant those specs kill becomes killed,
|
|
17
|
+
credited to the killing test in the JSON report, and counts in the score
|
|
18
|
+
and exit code. A survivor whose covering specs could not be run is listed
|
|
19
|
+
as unconfirmed, left out of the score and exit code, and ranked below
|
|
20
|
+
confirmed survivors. A map that is older than the tree and says nothing
|
|
21
|
+
about a survivor's lines keeps that survivor counted, with one line
|
|
22
|
+
naming how many. A covering spec outside the project root is never
|
|
23
|
+
loaded (#1286, #1304).
|
|
24
|
+
- `--color` and `--no-color` work on every text command, and a usage error
|
|
25
|
+
names the command that failed instead of showing `saboteur run` usage
|
|
26
|
+
(#1288, #1301).
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
|
|
30
|
+
- Only `saboteur run --all` records the whole-suite coverage map. A run
|
|
31
|
+
built from inferred specs no longer claims to be whole-suite. Without a
|
|
32
|
+
whole-suite map a scoped run prints one line saying survivors are
|
|
33
|
+
unconfirmed and how to build the map, and each survivor block says its
|
|
34
|
+
covering list is this run's selection only (#1286, #1300).
|
|
35
|
+
- A statement's gap label comes from all of its surviving mutants with a
|
|
36
|
+
fixed precedence: result unchecked, error path, branch, boundary,
|
|
37
|
+
effect, number, instead of from the first mutant (#1289, #1297).
|
|
38
|
+
- `baseline propose` records a mutant on a line no test runs as a gap
|
|
39
|
+
instead of an unexecuted blocker, so a tested method with one untested
|
|
40
|
+
line can still be proposed (#1287, #1299).
|
|
41
|
+
- A config-less Rails app's default `app/**/*.rb` include counts as a
|
|
42
|
+
full run, so the stale ignore entry warning fires there (#1255, #1296).
|
|
43
|
+
- `--quiet` no longer prints the missing-config notice, the coverage cache
|
|
44
|
+
miss or the map-build lines (#1264, #1293).
|
|
45
|
+
- Output polish: the run header names the `--subject` scope, the
|
|
46
|
+
no-spec note reads in the singular, untested subjects are named once
|
|
47
|
+
under "Not scored", and the replay example uses the top-ranked
|
|
48
|
+
survivor (#1291, #1298).
|
|
49
|
+
- Report test ids in `killedBy`, `coveredBy` and `testFiles` are numbered
|
|
50
|
+
by sorted test name, so the same tests keep the same ids across runs
|
|
51
|
+
(#1282, #1295). Covering tests are ordered by millisecond buckets, so
|
|
52
|
+
kill attribution is stable (#1274, #1283).
|
|
53
|
+
|
|
54
|
+
### Fixed
|
|
55
|
+
|
|
56
|
+
- An empty `--since` diff whose mutants all failed generation no longer
|
|
57
|
+
exits 0 under `--threshold`, and `--ignore-empty` no longer fails a run
|
|
58
|
+
without `--since` (#1265, #1266, #1284).
|
|
59
|
+
- `saboteur survivors` no longer drops an unknown keyword silently
|
|
60
|
+
(#1267, #1285).
|
|
61
|
+
|
|
62
|
+
### Corrected
|
|
63
|
+
|
|
64
|
+
- 0.12.0 said uncovered survivors are labeled `input_gap`, "separate from
|
|
65
|
+
assertion gaps" (#1254, #1157). Only `input_gap` exists, meaning no test
|
|
66
|
+
ran the mutant. There is no assertion-gap label yet (#1064).
|
|
67
|
+
- 0.11.2 said survivor blocks consult the whole-suite coverage map,
|
|
68
|
+
"ending false survivors from diff-scoped runs" (#1252). That change
|
|
69
|
+
only affects which covering tests a block lists. A survivor whose real
|
|
70
|
+
test sits in a spec file saboteur did not select can still be reported
|
|
71
|
+
as a survivor; see "Know the limits" in the README.
|
|
72
|
+
|
|
73
|
+
### Tests
|
|
74
|
+
|
|
75
|
+
- The worker pool and supervisor specs assert concurrency with a barrier
|
|
76
|
+
instead of a wall-clock bound (#1302, #1303).
|
|
77
|
+
|
|
78
|
+
## [0.12.1] - 2026-10-22
|
|
79
|
+
|
|
80
|
+
### Changed
|
|
81
|
+
|
|
82
|
+
- `--quiet` now stays quiet from startup: the start header, baseline
|
|
83
|
+
line, inference detail and progress bar no longer print. A `--since`
|
|
84
|
+
diff that selects no mutants exits 0 with a plain "nothing to gate"
|
|
85
|
+
note instead of failing a score threshold, and `--ignore-empty` is a
|
|
86
|
+
deprecated no-op. A diff with suppressed, omitted or undiscovered
|
|
87
|
+
mutants, or with an unverifiable scope, still gates (#1268, #1142,
|
|
88
|
+
#1149).
|
|
89
|
+
- `saboteur survivors` lists survivors ranked by default, matching
|
|
90
|
+
`run`, and owns its own command surface and help (#1269, #1150).
|
|
91
|
+
|
|
92
|
+
### Fixed
|
|
93
|
+
|
|
94
|
+
- An error-dominated run no longer prints a bare score: the terminal
|
|
95
|
+
closing line and the HTML report headline and summary say `n/a`, like
|
|
96
|
+
the terminal headline already did (#1275, #1271, #1276, #1272).
|
|
97
|
+
|
|
98
|
+
### Tests
|
|
99
|
+
|
|
100
|
+
- New end-to-end pins for per-path score gates, the `Score: n/a`
|
|
101
|
+
headline, the README get-started flow, parallel coverage parts and
|
|
102
|
+
the trace recorder, survivor triage labels, kills split by cause and
|
|
103
|
+
a docs-only `--since` diff under a threshold (#1270, #1273, #1278,
|
|
104
|
+
#1279, #1280, #1281, #1277, #1159, #1158).
|
|
105
|
+
|
|
9
106
|
## [0.12.0] - 2026-10-22
|
|
10
107
|
|
|
11
108
|
### Added
|
data/README.md
CHANGED
|
@@ -80,15 +80,19 @@ tree refuses with exit 2 and points at the generator. Engine gem with
|
|
|
80
80
|
assertion, 1 by a crash`), and an error-dominated run prints
|
|
81
81
|
`Score: n/a` instead of a number you would misread.
|
|
82
82
|
- 🔍 **Survivors you can act on.** Survivors group per statement with a
|
|
83
|
-
triage label (`likely_equivalent`, `arid`, `tool_limit`,
|
|
83
|
+
triage label (`likely_equivalent`, `arid`, `tool_limit`, `input_gap` when no test
|
|
84
|
+
ran the mutant, or a gap kind) and their
|
|
84
85
|
covering tests - split into the tests that ran against the mutant and
|
|
85
86
|
the ones that never did. `saboteur survivors` browses them read-only:
|
|
86
87
|
one digest shows the full diff and the exact replay command.
|
|
87
88
|
- 🧪 **Flakiness contained.** `--flaky-check survivors` re-runs each new
|
|
88
89
|
survivor in a reordered confirmation pass; a verdict the rerun disagrees
|
|
89
90
|
with becomes `unverified` and stays out of the score. With
|
|
90
|
-
`partial_coverage = "use"`, open
|
|
91
|
-
run against the covering specs
|
|
91
|
+
`partial_coverage = "use"`, open
|
|
92
|
+
survivors get one budgeted confirmation run against the covering specs
|
|
93
|
+
that never ran them when the map is built from every spec (`--all`, no
|
|
94
|
+
`--test`, one coverage part, cache on). Default inferred-spec runs, `pr`
|
|
95
|
+
included, skip it.
|
|
92
96
|
- 📊 **Reports for people and tools.** Terminal, HTML, a Stryker-compatible
|
|
93
97
|
JSON report, and a stable agent JSON API with a deprecation policy. The
|
|
94
98
|
GitHub Action posts survivors as review comments on your PR diff lines.
|
|
@@ -127,6 +131,15 @@ tree refuses with exit 2 and points at the generator. Engine gem with
|
|
|
127
131
|
- A survivor is review evidence, not proof of a bug in your tests. It can
|
|
128
132
|
be a missing test, an equivalent change, a flaky test, or a limit of
|
|
129
133
|
the tool. Read it before you write a test for it.
|
|
134
|
+
- A survivor can be false when the test that kills it lives in a spec
|
|
135
|
+
file saboteur did not select. Runs without `--test` pick specs by file
|
|
136
|
+
name and by the constants they read, so a test in a differently named
|
|
137
|
+
spec (say `spec/integration/flow_spec.rb` for `lib/ledger/router.rb`)
|
|
138
|
+
may never run against the mutant, and the "covered by ... ran, still
|
|
139
|
+
passed" lines list the specs that ran, plus any others marked "not run
|
|
140
|
+
here". If you know
|
|
141
|
+
another spec exercises the code, run again with `--test` naming it
|
|
142
|
+
before you trust the survivor.
|
|
130
143
|
- `--since` and `--incremental` are iteration aids. Run the whole project
|
|
131
144
|
on main and before a release.
|
|
132
145
|
- Sandboxing varies by platform and network access is never restricted.
|
|
@@ -21,6 +21,7 @@ module Saboteur
|
|
|
21
21
|
opts.on("--subject EXPR", "Whole subject filter; repeatable") { |value| add_subject(value) }
|
|
22
22
|
opts.on("--seed N", Integer, "Repeatable run seed") { |value| set(:seed, value) }
|
|
23
23
|
opts.on("--format FORMAT", "propose: terminal (default) or json") { |value| @format = value }
|
|
24
|
+
define_color_options(opts)
|
|
24
25
|
end
|
|
25
26
|
end
|
|
26
27
|
|
|
@@ -86,7 +87,11 @@ module Saboteur
|
|
|
86
87
|
survivors = payload.fetch("survivors")
|
|
87
88
|
@out.puts(" matched mutants: #{payload.fetch('candidates')} " \
|
|
88
89
|
"(#{plural(survivors, 'survivor')} to reason about)")
|
|
89
|
-
gaps = payload.fetch("gaps", [])
|
|
90
|
+
uncovered, gaps = payload.fetch("gaps", []).partition { |gap| gap.start_with?("no_coverage:") }
|
|
91
|
+
unless uncovered.empty?
|
|
92
|
+
@out.puts(" proposed with #{plural(uncovered.size, 'mutant')} on lines no test runs - " \
|
|
93
|
+
"the baseline cannot speak for them")
|
|
94
|
+
end
|
|
90
95
|
unless gaps.empty?
|
|
91
96
|
names = gaps.map { |gap| gap.sub(/\A(subject_skipped|omitted):/, "") }
|
|
92
97
|
@out.puts(" proposed with #{plural(names.size, 'untested subject')}: #{names.join(', ')} - " \
|
|
@@ -38,7 +38,10 @@ module Saboteur
|
|
|
38
38
|
def classify(baseline, current)
|
|
39
39
|
categories = CATEGORY_NAMES.to_h { |name| [name, []] }
|
|
40
40
|
previous = baseline.fetch("accepted").to_h { |entry| [entry.fetch("id"), entry] }
|
|
41
|
+
uncovered = uncovered_ids(current)
|
|
41
42
|
current.fetch("candidates").each do |entry|
|
|
43
|
+
next if uncovered.include?(entry.fetch("id"))
|
|
44
|
+
|
|
42
45
|
accepted = previous.delete(entry.fetch("id"))
|
|
43
46
|
name = category(entry, accepted)
|
|
44
47
|
categories.fetch(name) << entry if name
|
|
@@ -48,6 +51,11 @@ module Saboteur
|
|
|
48
51
|
categories
|
|
49
52
|
end
|
|
50
53
|
|
|
54
|
+
# Mutants on lines no test runs are listed as gaps, never as errors.
|
|
55
|
+
def uncovered_ids(current)
|
|
56
|
+
Completeness.uncovered_ids(current.fetch("gaps", []))
|
|
57
|
+
end
|
|
58
|
+
|
|
51
59
|
def category(entry, accepted)
|
|
52
60
|
case entry.fetch("status")
|
|
53
61
|
when "survived"
|
|
@@ -7,17 +7,24 @@ module Saboteur
|
|
|
7
7
|
class Completeness
|
|
8
8
|
EXECUTED = %w[killed survived timeout].freeze
|
|
9
9
|
# Soft gaps limit what the baseline can speak for; they never refuse a
|
|
10
|
-
# proposal. Everything else is a hard blocker.
|
|
11
|
-
|
|
10
|
+
# proposal. Everything else is a hard blocker. no_coverage: is a mutant
|
|
11
|
+
# on a line no test runs inside a tested subject (#1287).
|
|
12
|
+
SOFT_PREFIXES = %w[subject_skipped: omitted: no_coverage:].freeze
|
|
12
13
|
attr_reader :issues
|
|
13
14
|
|
|
15
|
+
# The mutant ids named by no_coverage: gaps.
|
|
16
|
+
def self.uncovered_ids(gaps)
|
|
17
|
+
gaps.filter_map { |gap| gap.delete_prefix("no_coverage:") if gap.start_with?("no_coverage:") }
|
|
18
|
+
end
|
|
19
|
+
|
|
14
20
|
def self.soft_gap?(issue)
|
|
15
21
|
SOFT_PREFIXES.any? { |prefix| issue.start_with?(prefix) }
|
|
16
22
|
end
|
|
17
23
|
|
|
18
24
|
def initialize(planned:, results:, subjects:, stable_suite:, stable_inputs:, skipped_subjects:, omitted:,
|
|
19
|
-
generation_issues:)
|
|
25
|
+
generation_issues:, uncovered: [])
|
|
20
26
|
@issues = []
|
|
27
|
+
@uncovered = uncovered
|
|
21
28
|
issues << "unchanged_suite_incomplete" unless stable_suite
|
|
22
29
|
issues << "inputs_unverified" unless stable_inputs
|
|
23
30
|
validate_inventory(planned, subjects)
|
|
@@ -66,7 +73,10 @@ module Saboteur
|
|
|
66
73
|
def validate_results(results)
|
|
67
74
|
results.each do |entry|
|
|
68
75
|
status = entry.fetch("status")
|
|
69
|
-
|
|
76
|
+
next if EXECUTED.include?(status)
|
|
77
|
+
|
|
78
|
+
id = entry.fetch("id")
|
|
79
|
+
issues << (@uncovered.include?(id) ? "no_coverage:#{id}" : "unexecuted:#{id}:#{status}")
|
|
70
80
|
end
|
|
71
81
|
end
|
|
72
82
|
end
|
|
@@ -47,11 +47,18 @@ module Saboteur
|
|
|
47
47
|
entry.slice("id", "context").merge("reason" => reasons)
|
|
48
48
|
end
|
|
49
49
|
{ "format" => Manifest::FORMAT, "identity_version" => Identity::VERSION,
|
|
50
|
-
"metadata" => metadata, "candidates" =>
|
|
50
|
+
"metadata" => metadata, "candidates" => executed_candidates, "accepted" => accepted }
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
private
|
|
54
54
|
|
|
55
|
+
# A mutant on a line no test runs is a gap in the proposal's issues,
|
|
56
|
+
# not a candidate: the baseline has no verdict to compare it against.
|
|
57
|
+
def executed_candidates
|
|
58
|
+
uncovered = Completeness.uncovered_ids(document.fetch("gaps"))
|
|
59
|
+
document.fetch("candidates").reject { |entry| uncovered.include?(entry.fetch("id")) }
|
|
60
|
+
end
|
|
61
|
+
|
|
55
62
|
def array!(value, name)
|
|
56
63
|
raise ConfigError, "baseline #{name}" unless value.is_a?(Array)
|
|
57
64
|
end
|
|
@@ -43,6 +43,7 @@ module Saboteur
|
|
|
43
43
|
errors = generation_issues(generation, results) + context_issues(observed) + discovery_issues(discovery)
|
|
44
44
|
suppressions = suppression_scope(inline_suppressions)
|
|
45
45
|
errors += suppression_issues(suppressions)
|
|
46
|
+
@uncovered = uncovered_ids(results)
|
|
46
47
|
completeness = assess(observed, stable_suite, skipped_subjects, omissions, errors)
|
|
47
48
|
document = { "compatibility" => compatibility, "snapshot" => @snapshot.to_h, "subjects" => @subjects,
|
|
48
49
|
"candidates" => observed, "planned" => @planned,
|
|
@@ -83,13 +84,21 @@ module Saboteur
|
|
|
83
84
|
end
|
|
84
85
|
end
|
|
85
86
|
|
|
87
|
+
# Mutants skipped only because no test runs their line.
|
|
88
|
+
def uncovered_ids(results)
|
|
89
|
+
uncovered = results.select { |result| result.reason&.start_with?("no coverage") }
|
|
90
|
+
uncovered.map do |result|
|
|
91
|
+
Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.fetch("id")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
86
95
|
def assess(observed, stable_suite, skipped_subjects, omissions, errors)
|
|
87
96
|
skipped = skipped_subjects.map { |subject, reason| "#{subject.id}:#{reason}" }
|
|
88
97
|
Completeness.new(planned: @planned.map { |entry| entry.fetch("id") }, results: observed,
|
|
89
98
|
subjects: @subjects, stable_suite: stable_suite,
|
|
90
99
|
stable_inputs: @fingerprint.verify(@snapshot).unchanged, skipped_subjects: skipped,
|
|
91
100
|
omitted: omissions.map { |entry| "#{entry.subject}:#{entry.count}" },
|
|
92
|
-
generation_issues: errors)
|
|
101
|
+
generation_issues: errors, uncovered: @uncovered)
|
|
93
102
|
end
|
|
94
103
|
|
|
95
104
|
def compatibility
|
|
@@ -13,6 +13,8 @@ module Saboteur
|
|
|
13
13
|
# Opt-in orchestration: every check/proposal runs fresh; acceptance only
|
|
14
14
|
# edits reasons after verifying private execution evidence and current inputs.
|
|
15
15
|
class Workflow
|
|
16
|
+
UNEXECUTED = /\Aunexecuted:([0-9a-f]{64}):(\w+)\z/
|
|
17
|
+
|
|
16
18
|
def initialize(config:, err: $stderr)
|
|
17
19
|
@config = config
|
|
18
20
|
@err = err
|
|
@@ -50,20 +52,30 @@ module Saboteur
|
|
|
50
52
|
[Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.fetch("id"), result]
|
|
51
53
|
end
|
|
52
54
|
blockers.each { |issue| @err.puts("saboteur: baseline proposal blocker: #{describe_blocker(issue, by_id)}") }
|
|
53
|
-
|
|
54
|
-
@err.puts("saboteur: cover the skipped code with specs or narrow include, then propose again")
|
|
55
|
-
end
|
|
55
|
+
advise_unexecuted(blockers, by_id)
|
|
56
56
|
raise Error, "baseline proposal needs a proposable run"
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
# Narrowing include cannot add line coverage, so a blocker whose only
|
|
60
|
+
# cause is no coverage gets the specs advice alone.
|
|
61
|
+
def advise_unexecuted(blockers, by_id)
|
|
62
|
+
unexecuted = blockers.grep(/\Aunexecuted:/)
|
|
63
|
+
return if unexecuted.empty?
|
|
64
|
+
|
|
65
|
+
results = unexecuted.map { |issue| by_id[UNEXECUTED.match(issue)&.[](1)] }
|
|
66
|
+
coverage_only = results.all? { |result| result&.reason&.start_with?("no coverage") }
|
|
67
|
+
advice = coverage_only ? "specs" : "specs or narrow include"
|
|
68
|
+
@err.puts("saboteur: cover the skipped code with #{advice}, then propose again")
|
|
69
|
+
end
|
|
70
|
+
|
|
59
71
|
def describe_blocker(issue, by_id)
|
|
60
|
-
match =
|
|
72
|
+
match = UNEXECUTED.match(issue)
|
|
61
73
|
result = match && by_id[match[1]]
|
|
62
74
|
return issue unless result
|
|
63
75
|
|
|
64
76
|
detail = "#{result.subject.id} #{result.mutation.operator}"
|
|
65
77
|
detail += " - #{result.reason}" if result.reason
|
|
66
|
-
"
|
|
78
|
+
"unexecuted:#{result.digest}:#{match[2]} (#{detail})"
|
|
67
79
|
end
|
|
68
80
|
|
|
69
81
|
def propose(proposal:, destination:, previous: nil)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# The colour switches every command that prints text accepts (#1288).
|
|
7
|
+
# --no-color stays as a deprecated alias for --color=never.
|
|
8
|
+
module ColorOption
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def define_color_options(opts)
|
|
12
|
+
opts.on("--color [MODE]", %w[always never auto],
|
|
13
|
+
"Colour the text report: always, never or auto (default: auto)") do |value|
|
|
14
|
+
@color = (value || :always).to_sym
|
|
15
|
+
end
|
|
16
|
+
opts.on("--no-color", "Deprecated alias for --color=never") { @color = :never }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Consume leading colour switches for commands that otherwise take
|
|
20
|
+
# positional arguments only (help, explain).
|
|
21
|
+
def consume_color_options
|
|
22
|
+
OptionParser.new { |opts| define_color_options(opts) }.order!(@argv)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -15,7 +15,7 @@ module Saboteur
|
|
|
15
15
|
def exit_code(report)
|
|
16
16
|
return EXIT_BUDGET if budget_exhausted?(report)
|
|
17
17
|
|
|
18
|
-
return EXIT_OK if
|
|
18
|
+
return EXIT_OK if empty_clean_diff?(report)
|
|
19
19
|
|
|
20
20
|
return EXIT_TOOL if untested_partial?(report)
|
|
21
21
|
|
|
@@ -29,14 +29,19 @@ module Saboteur
|
|
|
29
29
|
gates.failed?(report, @err, record_cause: !quality_failed) || quality_failed ? EXIT_SURVIVORS : EXIT_OK
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
#
|
|
33
|
-
# a
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
# A --since diff that selected no mutants exits 0 with a plain note,
|
|
33
|
+
# with or without --ignore-empty: a docs-only change has no score, so a
|
|
34
|
+
# threshold must not fail it (#1142). Suppressed, omitted or
|
|
35
|
+
# undiscovered mutants still gate, and so do mutants dropped because they
|
|
36
|
+
# did not parse (#1265) - the selection must be honestly empty, not just
|
|
37
|
+
# empty-handed.
|
|
38
|
+
def empty_clean_diff?(report)
|
|
39
|
+
return false unless config["since"] && report.results.empty?
|
|
40
|
+
return false if unverifiable_scope?(report)
|
|
36
41
|
return false unless report.omissions.empty? && report.inline_suppressions.empty? &&
|
|
37
|
-
report.discovery_failures.empty?
|
|
42
|
+
report.discovery_failures.empty? && report.invalid_mutations.zero?
|
|
38
43
|
|
|
39
|
-
@err.puts("saboteur: no mutants in the diff - nothing to gate
|
|
44
|
+
@err.puts("saboteur: no mutants in the diff - nothing to gate")
|
|
40
45
|
true
|
|
41
46
|
end
|
|
42
47
|
|
|
@@ -15,6 +15,7 @@ module Saboteur
|
|
|
15
15
|
private
|
|
16
16
|
|
|
17
17
|
def explain_command
|
|
18
|
+
consume_color_options
|
|
18
19
|
name = @argv.shift
|
|
19
20
|
raise UsageError, "usage: saboteur explain <operator>" if name.nil?
|
|
20
21
|
|
|
@@ -27,7 +28,7 @@ module Saboteur
|
|
|
27
28
|
return EXIT_USAGE
|
|
28
29
|
end
|
|
29
30
|
|
|
30
|
-
@out.puts(render_explain(entry, palette: Reporter::Palette.for(@out, mode: :auto)))
|
|
31
|
+
@out.puts(render_explain(entry, palette: Reporter::Palette.for(@out, mode: @color || :auto)))
|
|
31
32
|
EXIT_OK
|
|
32
33
|
rescue UsageError => e
|
|
33
34
|
@err.puts("saboteur: #{e.message}")
|
|
@@ -26,8 +26,19 @@ module Saboteur
|
|
|
26
26
|
|
|
27
27
|
def usage_error(message = nil)
|
|
28
28
|
@err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, message)) if message
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
usage = command_usage
|
|
30
|
+
@err.puts(usage)
|
|
31
|
+
agent_failure(message || usage, EXIT_USAGE)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The usage line of the command that failed, never run's for another
|
|
35
|
+
# command (#1288).
|
|
36
|
+
def command_usage
|
|
37
|
+
banner = { "doctor" => :doctor_parser, "baseline" => :baseline_parser, "badge" => :badge_parser,
|
|
38
|
+
"trend" => :trend_parser }[@command]
|
|
39
|
+
return USAGE unless banner
|
|
40
|
+
|
|
41
|
+
"#{send(banner).banner}\nRun 'saboteur help #{@command}' for every option."
|
|
31
42
|
end
|
|
32
43
|
|
|
33
44
|
def agent? = @format == "agent"
|
|
@@ -20,6 +20,7 @@ module Saboteur
|
|
|
20
20
|
opts.on("--mark-partial", "Badge an incomplete run, labeled as partial") { @mark_partial = true }
|
|
21
21
|
opts.on("--high PERCENT", Float, "brightgreen at or above (default #{DEFAULT_HIGH.to_i})") { |v| @high = v }
|
|
22
22
|
opts.on("--low PERCENT", Float, "yellow at or above (default #{DEFAULT_LOW.to_i})") { |v| @low = v }
|
|
23
|
+
define_color_options(opts)
|
|
23
24
|
end
|
|
24
25
|
end
|
|
25
26
|
|
|
@@ -27,6 +28,7 @@ module Saboteur
|
|
|
27
28
|
OptionParser.new do |opts|
|
|
28
29
|
opts.banner = "Usage: saboteur trend [--history PATH]"
|
|
29
30
|
opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
|
|
31
|
+
define_color_options(opts)
|
|
30
32
|
end
|
|
31
33
|
end
|
|
32
34
|
|
|
@@ -29,10 +29,7 @@ module Saboteur
|
|
|
29
29
|
@verbose = true
|
|
30
30
|
set(:verbose, true)
|
|
31
31
|
end
|
|
32
|
-
opts
|
|
33
|
-
"Colour the text report: always, never or auto (default: auto)") do |v|
|
|
34
|
-
@color = (v || :always).to_sym
|
|
35
|
-
end
|
|
32
|
+
define_color_options(opts)
|
|
36
33
|
|
|
37
34
|
# PR6: CI and pre-push output - the findings and the verdict only.
|
|
38
35
|
opts.on("--quiet", "Text report: print only the findings and the verdict;",
|
|
@@ -46,7 +43,6 @@ module Saboteur
|
|
|
46
43
|
# top three; --details restores a block for every finding.
|
|
47
44
|
opts.on("--details", "Text report: full blocks for every finding;",
|
|
48
45
|
"default: one-line list plus the top 3 blocks") { set(:details, true) }
|
|
49
|
-
opts.on("--no-color", "Deprecated alias for --color=never") { @color = :never }
|
|
50
46
|
end
|
|
51
47
|
end
|
|
52
48
|
end
|
|
@@ -22,7 +22,7 @@ module Saboteur
|
|
|
22
22
|
# there is no init command - the Rails generator writes the config.
|
|
23
23
|
def announce_missing_config_file(palette)
|
|
24
24
|
# @format, not agent_requested?: parse! has already eaten --format.
|
|
25
|
-
return if @format == "agent" || config.config_file_present
|
|
25
|
+
return if @format == "agent" || options[:quiet] || config.config_file_present
|
|
26
26
|
|
|
27
27
|
path = options[:config] || Config::FILE
|
|
28
28
|
notice =
|
|
@@ -23,7 +23,7 @@ module Saboteur
|
|
|
23
23
|
replay_command: survivor_replay_command(report, engine),
|
|
24
24
|
all_mutants: options[:all_mutants] || false, details: options[:details] || false,
|
|
25
25
|
exit_code: code, previous_score: previous_score, scope: scope_label,
|
|
26
|
-
whole_suite_map: whole_suite_map_for(engine),
|
|
26
|
+
whole_suite_map: whole_suite_map_for(engine, report),
|
|
27
27
|
run_map: engine&.run_coverage_map,
|
|
28
28
|
baseline_gate: (baseline_gate(report) if config["baseline"])).call(report)
|
|
29
29
|
Reporter::Replay.new(@out).call(report) if options[:id]
|
|
@@ -40,14 +40,6 @@ module Saboteur
|
|
|
40
40
|
code
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
# No engine or no cache: no whole-suite map can exist, so the survivor
|
|
44
|
-
# block keeps its legacy covered-by line over an impossible build (review C1).
|
|
45
|
-
def whole_suite_map_for(engine)
|
|
46
|
-
return engine.whole_suite_coverage_map if engine&.whole_suite_map_lookup?
|
|
47
|
-
|
|
48
|
-
Reporter::SurvivorList::NO_WHOLE_SUITE_LOOKUP
|
|
49
|
-
end
|
|
50
|
-
|
|
51
43
|
def agent_run(arguments)
|
|
52
44
|
tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
|
|
53
45
|
# The machine path is exempt from the noise diet: quiet_start: false
|
|
@@ -81,7 +73,7 @@ module Saboteur
|
|
|
81
73
|
|
|
82
74
|
# Text mode only: the engine prints a per-subject marks line when
|
|
83
75
|
# --verbose is on (#262); --agent output stays as it is.
|
|
84
|
-
engine = Engine.new(config, **arguments, verbose: @verbose || false)
|
|
76
|
+
engine = Engine.new(config, **arguments, verbose: @verbose || false, quiet: @quiet || false)
|
|
85
77
|
text_run(engine.call, engine)
|
|
86
78
|
end
|
|
87
79
|
|
|
@@ -22,8 +22,9 @@ module Saboteur
|
|
|
22
22
|
opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
|
|
23
23
|
opts.on("--shard K/N", "Run one of N deterministic candidate shards (for example 1/4)") { |v| set(:shard, v) }
|
|
24
24
|
opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
|
|
25
|
-
opts.on("--ignore-empty", "
|
|
26
|
-
|
|
25
|
+
opts.on("--ignore-empty", "Deprecated no-op: the no-mutant diff case always exits 0 (#1142)") do
|
|
26
|
+
set(:ignore_empty, true)
|
|
27
|
+
end
|
|
27
28
|
flaky_desc = "Confirm a fresh verdict (survivors, decisive) with one reordered rerun"
|
|
28
29
|
opts.on("--flaky-check MODE", %w[off survivors decisive], flaky_desc) { |v| set(:flaky_check, v) }
|
|
29
30
|
opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
|