saboteur 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +115 -0
- data/README.md +34 -6
- data/lib/saboteur/attribute_finder.rb +113 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/run_budget_options.rb +3 -1
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +4 -2
- data/lib/saboteur/config.rb +4 -1
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/custom_map.rb +12 -1
- data/lib/saboteur/dirty_tree_warning.rb +5 -3
- data/lib/saboteur/doctor.rb +4 -0
- data/lib/saboteur/engine/baseline_guard.rb +76 -10
- data/lib/saboteur/engine/boot.rb +7 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +1 -0
- data/lib/saboteur/engine/coverage_store.rb +25 -3
- data/lib/saboteur/engine/diagnostics.rb +1 -1
- data/lib/saboteur/engine/gated_run.rb +3 -1
- data/lib/saboteur/engine/load_coverage.rb +18 -0
- data/lib/saboteur/engine/pipelines.rb +8 -0
- data/lib/saboteur/engine/run_budget.rb +5 -2
- data/lib/saboteur/engine/sandbox_policy.rb +21 -3
- data/lib/saboteur/engine/scheduler.rb +32 -24
- data/lib/saboteur/engine/selection_announcement.rb +8 -3
- data/lib/saboteur/engine/subject_selection.rb +115 -31
- data/lib/saboteur/engine/test_guess.rb +106 -16
- data/lib/saboteur/engine.rb +3 -2
- data/lib/saboteur/execution/external_runner.rb +18 -0
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +10 -4
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +2 -1
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/integration/rspec.rb +99 -11
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +3 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/operator_examples.rb +1 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +1 -0
- data/lib/saboteur/preview.rb +5 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +20 -5
- data/lib/saboteur/reporter/terminal.rb +16 -1
- data/lib/saboteur/selection/coverage_map.rb +17 -2
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +1 -0
- metadata +5 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0a4dcacd09e6b4ca97537446ae6cbc73b13fb2c6dabcc26f22ea26784e869151
|
|
4
|
+
data.tar.gz: 83c7e9f8121a1193c518cbfcba50b825d13b0e08ccd90f3085f88d932855fd03
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a9baa6f9af8e3f2ecd0e2fd661104550e23b7b5e2f368d4cd996b5648994537888d88ba1130477525b93d4df13a756585eed3f9ffb27f309057e7452d069a982
|
|
7
|
+
data.tar.gz: 782690794b535288178addb949cf7b0e530e19d4e55acd20c837903f4ddb8818581b42cacfc5af1d1ccb7ba4a6dd9e0cbd84a5059f143407f99195e2ed14a046
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,121 @@ All notable changes are documented here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.8.1] - 2026-10-18
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Positional call-argument swap operator (#653): swaps adjacent positional
|
|
14
|
+
arguments at call sites, catching specs that do not pin argument order.
|
|
15
|
+
- After the first timed baseline pass, saboteur projects the remaining
|
|
16
|
+
baseline+coverage cost against `--max-duration` and warns up front when
|
|
17
|
+
the budget barely fits or is exceeded, naming how much time mutants get
|
|
18
|
+
and the remedies (#637). A `--max-duration` run that ends with exit 4
|
|
19
|
+
and 0 mutants now headlines "no mutants ran", and the flag's help states
|
|
20
|
+
what the clock covers: it starts at engine entry and includes the
|
|
21
|
+
baseline and coverage phases.
|
|
22
|
+
- fiddle is no longer a hard dependency: it loads lazily through one door
|
|
23
|
+
that refuses with every working fix named when it is missing, and the
|
|
24
|
+
sandbox layers fail closed on refusal (#639).
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- The DCO checks no longer disagree: CI's check-dco, preflight, and the
|
|
29
|
+
panel gate all require the Signed-off-by line to be a real git trailer
|
|
30
|
+
after a blank line (#646).
|
|
31
|
+
- When a changed file matches no spec by id hint, saboteur names the exact
|
|
32
|
+
ids it looked for (`warn_unmatched_whole_file`, `--since` scoped) (#635).
|
|
33
|
+
- `saboteur pr --since` on a spec-only or docs-only diff no longer warns
|
|
34
|
+
that "the whole suite runs" - it announces there is nothing to test,
|
|
35
|
+
matching the zero-subject run that follows (#607).
|
|
36
|
+
- On macOS the degraded-sandbox message no longer claims the kernel "lacks
|
|
37
|
+
Landlock (needs Linux 6.2+)"; it says macOS has no Landlock and points
|
|
38
|
+
at `c.sandbox = "seatbelt"` (#667).
|
|
39
|
+
- The preset banner shows the effective flags, including explicit
|
|
40
|
+
`--since` refs and flag=value spellings (#666).
|
|
41
|
+
- Running saboteur without rspec-core in the bundle (for example a
|
|
42
|
+
Minitest project) fails as a usage error (exit 2) naming docs/minitest.md,
|
|
43
|
+
instead of an internal error pointing at the RSpec-only `c.integration`
|
|
44
|
+
remedy (#610). A baseline that fails with NoMethodError in a suite whose
|
|
45
|
+
spec_helper requires minitest now warns the suite looks like Minitest
|
|
46
|
+
running under RSpec, with the symptom and count (#626).
|
|
47
|
+
- Test-file guessing looks for specs below a flattened namespace
|
|
48
|
+
(rubocop-ast style layouts) (#586), and when a layout strips directories
|
|
49
|
+
between source and spec trees it picks the unique spec by parent
|
|
50
|
+
directory and basename instead of failing or guessing wrong (#600).
|
|
51
|
+
- A `--subject` pattern that names an attr_reader, attr_writer or
|
|
52
|
+
attr_accessor method no longer says "matches no subject"; the method
|
|
53
|
+
exists but has no body to mutate, and the message says so (#373).
|
|
54
|
+
- Load errors in the target app's boot are reported with the tool's own
|
|
55
|
+
frames filtered out, with `--verbose` restoring the full backtrace (#608).
|
|
56
|
+
- If a spec file loaded Ruby code before saboteur started coverage,
|
|
57
|
+
whole-suite runs no longer report "5 subjects, 0 mutants"; saboteur
|
|
58
|
+
names up to 5 preloaded files with cause and remedy and exits 3. With
|
|
59
|
+
`--test` it warns and continues (#686).
|
|
60
|
+
- An empty or docs-only `--since` range exits 0 with a "nothing to test"
|
|
61
|
+
notice instead of failing (#659), and a run no longer floods stderr with
|
|
62
|
+
Ruby's "possibly useless use of nil in void context" compile warnings
|
|
63
|
+
when the host suite enables warnings (#656).
|
|
64
|
+
- Running saboteur in a project with no RSpec suite stops with a single
|
|
65
|
+
line naming the problem and pointing to docs/minitest.md, instead of one
|
|
66
|
+
line per source file; specs outside spec/ fall back to the per-file stop
|
|
67
|
+
with `--test`/`--all`. Exit code unchanged (2) (#606).
|
|
68
|
+
- Whole-file targets no longer error "already initialized constant" in
|
|
69
|
+
strict-warning suites when the app lazily requires the same file (#632).
|
|
70
|
+
- Doctor's macOS degraded-sandbox output names the way out:
|
|
71
|
+
`c.sandbox = "seatbelt"` with the guide link, or `c.sandbox = false` (#667).
|
|
72
|
+
- Load-time vetting no longer rejects `class << self` or `/re/.freeze`,
|
|
73
|
+
and the constant-skip warning cites a one-line excerpt plus line instead
|
|
74
|
+
of whole class bodies (#664). Vetting clears Struct/Data factories,
|
|
75
|
+
public_constant/private_constant, and lambda literals so those files
|
|
76
|
+
keep their mutation subjects (#537).
|
|
77
|
+
- A spec load that quits because the suite auto-starts SimpleCov/SingleCov
|
|
78
|
+
names the suspect and both guard styles, instead of an opaque "No
|
|
79
|
+
examples found" (#645).
|
|
80
|
+
- A sqlite :memory: database with a schema but zero rows hints at
|
|
81
|
+
require-time seed data lost across the fork, with the
|
|
82
|
+
before(:suite)/after_fork remedy and the `c.hooks` wiring (#594).
|
|
83
|
+
- The early-stopped summary headline reads selected versus attempted, for
|
|
84
|
+
example "57 mutants selected, 4 attempted" (#582).
|
|
85
|
+
- The dirty-tree warning headline says "uncommitted or untracked change(s)"
|
|
86
|
+
so a tree of only scratch files is not read as edits (#604).
|
|
87
|
+
- Custom-map load errors name unsupported template syntax before glob
|
|
88
|
+
validation, literal map paths are preserved, and mixing numbered and
|
|
89
|
+
named map templates is rejected at load with a clear error (#642).
|
|
90
|
+
- The sqlite in-memory database fork-loss hint includes the exact
|
|
91
|
+
`c.hooks` configuration needed to load the suggested after-fork hooks
|
|
92
|
+
file (#593).
|
|
93
|
+
- A subject the `--max-duration` budget stops mid-list no longer drops the
|
|
94
|
+
kills it already measured: the truncated subject's completed kills are
|
|
95
|
+
confirmed uncapped (#588).
|
|
96
|
+
- Shell-exported RUBYLIB entries that bundler's scrub drops are restored
|
|
97
|
+
(#584), `spec/` is on the load path for non-Rails boots too (#615), and
|
|
98
|
+
RSpec core loads before configured boot helpers so non-Rails helpers can
|
|
99
|
+
call RSpec.configure (#616).
|
|
100
|
+
- `--test-command` warns when one ruby is handed several test files: ruby
|
|
101
|
+
executes only the first, so the rest never ran and their mutants
|
|
102
|
+
silently survived (#611).
|
|
103
|
+
- Dry-run excludes statically unsupported subjects, such as methods
|
|
104
|
+
defined inside blocks, from planned mutant totals and reports the skip
|
|
105
|
+
reason (#651).
|
|
106
|
+
- Non-String integration assignments fail with a file-and-line
|
|
107
|
+
configuration error before fingerprinting or boot (#614).
|
|
108
|
+
- Baseline failures outside examples say so and show the log tail instead
|
|
109
|
+
of an empty failed-example list (#581).
|
|
110
|
+
|
|
111
|
+
### Documentation
|
|
112
|
+
|
|
113
|
+
- The flaky-tests and survivors guides catch up with 0.8.0 (#683), and
|
|
114
|
+
design.md explains the load-time vetting safelist for constant replay
|
|
115
|
+
under `--since`, including what it cannot account for (#537).
|
|
116
|
+
- Docs drift: the pinned saboteur-vs-mutineer benchmark report names the
|
|
117
|
+
version it measured (0.1.0), and the compare page scopes its kill
|
|
118
|
+
re-check claim to fresh kills (#657).
|
|
119
|
+
- The README quick start covers the inference-miss exit 2 and its
|
|
120
|
+
recovery, the install block leads with Bundler, the quick-start anchor
|
|
121
|
+
matches its heading, and a no-warranty disclaimer joins the license
|
|
122
|
+
section (#658, #657, #599).
|
|
123
|
+
|
|
9
124
|
## [0.8.0] - 2026-10-18
|
|
10
125
|
|
|
11
126
|
### Added
|
data/README.md
CHANGED
|
@@ -21,26 +21,44 @@ code, so a broken suite never counts as a kill.
|
|
|
21
21
|
**Docs, benchmarks and guides live on the site:
|
|
22
22
|
[vamsipavanmahesh.github.io/saboteur](https://vamsipavanmahesh.github.io/saboteur/)**
|
|
23
23
|
|
|
24
|
-
<a id="
|
|
24
|
+
<a id="30-second-quick-start"></a>
|
|
25
25
|
## 30-second quick start
|
|
26
26
|
|
|
27
27
|
You need Ruby 3.3 or newer, RSpec, and Linux or macOS with `fork`.
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
gem
|
|
29
|
+
Prerequisite: the sandbox layers use fiddle, which ships with Ruby
|
|
30
|
+
(a default gem on Ruby 3.x). On Ruby 4.0+, where fiddle is a bundled
|
|
31
|
+
gem, add `gem "fiddle"` to your Gemfile. If saboteur refuses a run
|
|
32
|
+
because fiddle cannot load: reinstall Ruby, run `gem install fiddle`,
|
|
33
|
+
or install libffi headers (libffi-dev on Debian/Ubuntu, libffi-devel
|
|
34
|
+
on Fedora/RHEL) and reinstall fiddle. libffi headers are required only
|
|
35
|
+
when Ruby's shipped fiddle is missing or must be installed standalone.
|
|
32
36
|
|
|
33
|
-
|
|
37
|
+
```sh
|
|
38
|
+
# Add it to your project's Gemfile:
|
|
34
39
|
bundle add saboteur --group test
|
|
35
40
|
|
|
36
|
-
#
|
|
41
|
+
# Then:
|
|
37
42
|
bundle exec saboteur run
|
|
38
43
|
```
|
|
39
44
|
|
|
45
|
+
Outside a Bundler project, `gem install saboteur` gives the same
|
|
46
|
+
executable - run it there as plain `saboteur run`, without
|
|
47
|
+
`bundle exec`.
|
|
48
|
+
|
|
40
49
|
The first run builds a coverage map of which tests touch which code. The
|
|
41
50
|
map is cached and reused while your code and tests stay unchanged, so
|
|
42
51
|
later runs start fast.
|
|
43
52
|
|
|
53
|
+
If the first run stops with exit 2 and `no spec found for ...`, saboteur
|
|
54
|
+
could not map a source file to its spec: it matches by path first, then
|
|
55
|
+
by parent directory and basename when the layout strips directories,
|
|
56
|
+
and it refuses to guess wrong. Point it at the specs yourself - set
|
|
57
|
+
`test_selection = :all` in `config/saboteur.rb` to run the whole suite
|
|
58
|
+
against each mutant, pass `--test spec/...` to name the specs, or
|
|
59
|
+
declare a custom map
|
|
60
|
+
([monorepos and the custom map](docs/guides/monorepo-and-custom-map.md)).
|
|
61
|
+
|
|
44
62
|
On a branch, `bundle exec saboteur pr` mutates only the methods you
|
|
45
63
|
changed relative to `main`, with one baseline pass and a ten-minute
|
|
46
64
|
budget. If the budget stops the run, it exits 4 and prints `partial: X
|
|
@@ -247,6 +265,16 @@ on the site and [docs/benchmark](docs/benchmark/) in this repo.
|
|
|
247
265
|
- [Contributing](CONTRIBUTING.md) - every commit needs a DCO `Signed-off-by` line
|
|
248
266
|
- [Design notes](docs/design.md)
|
|
249
267
|
|
|
268
|
+
## Disclaimer
|
|
269
|
+
|
|
270
|
+
saboteur is provided "as is", without warranty of any kind, express or
|
|
271
|
+
implied. In no event shall the authors or copyright holders be liable
|
|
272
|
+
for any claim, damages or other liability arising from its use -
|
|
273
|
+
including damage caused by mutating your code or by running unusual
|
|
274
|
+
DSLs and metaprogramming-heavy code. Mutation testing runs your test
|
|
275
|
+
suite against deliberately broken code; run it on a clean, committed
|
|
276
|
+
tree or a CI checkout. The full text is in [LICENSE](LICENSE).
|
|
277
|
+
|
|
250
278
|
## License
|
|
251
279
|
|
|
252
280
|
[MIT](LICENSE).
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "subject"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
# Finds the methods attribute macros generate - attr_reader, attr_writer,
|
|
9
|
+
# attr_accessor - so a --subject pattern naming one can be answered with
|
|
10
|
+
# "defined by attr_reader, no body to mutate" instead of the
|
|
11
|
+
# pattern-format hint (#373). The generated method has no body saboteur
|
|
12
|
+
# can mutate, so it is never a mutation subject. Only a receiverless
|
|
13
|
+
# call with literal symbol or string names defines anything nameable;
|
|
14
|
+
# like SubjectFinder, a macro inside a block binds to an unknowable
|
|
15
|
+
# self and is skipped.
|
|
16
|
+
class AttributeFinder
|
|
17
|
+
# One generated method: id follows Subject#id ("Owner#name",
|
|
18
|
+
# "Owner.name", "Owner#name="), macro is the call that made it.
|
|
19
|
+
Found = Data.define(:id, :macro)
|
|
20
|
+
|
|
21
|
+
MACROS = %i[attr_reader attr_writer attr_accessor].freeze
|
|
22
|
+
|
|
23
|
+
def call(file)
|
|
24
|
+
result = Prism.parse(File.binread(file), filepath: file)
|
|
25
|
+
return [] if result.failure?
|
|
26
|
+
|
|
27
|
+
found = []
|
|
28
|
+
walk(result.value, [], found)
|
|
29
|
+
found
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def walk(node, nesting, out, in_block: false)
|
|
35
|
+
return if node.nil?
|
|
36
|
+
|
|
37
|
+
case node
|
|
38
|
+
when Prism::BlockNode, Prism::LambdaNode
|
|
39
|
+
walk(node.body, nesting, out, in_block: true)
|
|
40
|
+
# A keyword class or module re-enters the lexical scope: a macro
|
|
41
|
+
# inside binds on the class even when the class sits in a block
|
|
42
|
+
# (mirrors ConstantSubjectFinder).
|
|
43
|
+
when Prism::ModuleNode
|
|
44
|
+
walk(node.body, nesting + [Scope.new(kind: :module, path: node.constant_path.slice)], out)
|
|
45
|
+
when Prism::ClassNode
|
|
46
|
+
walk(node.body, nesting + [Scope.new(kind: :class, path: node.constant_path.slice)], out)
|
|
47
|
+
when Prism::SingletonClassNode
|
|
48
|
+
walk_singleton_class(node, nesting, out)
|
|
49
|
+
when Prism::CallNode
|
|
50
|
+
record_macro(node, nesting, out, in_block: in_block)
|
|
51
|
+
node.compact_child_nodes.each { |child| walk(child, nesting, out, in_block: in_block) }
|
|
52
|
+
else
|
|
53
|
+
node.compact_child_nodes.each { |child| walk(child, nesting, out, in_block: in_block) }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# `class << self`: attribute macros inside define singleton methods on
|
|
58
|
+
# the enclosing class (mirrors SubjectFinder's singleton defs).
|
|
59
|
+
def walk_singleton_class(node, nesting, out)
|
|
60
|
+
return unless node.expression.is_a?(Prism::SelfNode)
|
|
61
|
+
return if nesting.empty? || nesting.last.sclass?
|
|
62
|
+
|
|
63
|
+
walk(node.body, nesting + [Scope.new(kind: :sclass, path: "self")], out)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def record_macro(node, nesting, out, in_block:)
|
|
67
|
+
return unless MACROS.include?(node.name) && node.receiver.nil? && node.block.nil?
|
|
68
|
+
return if in_block
|
|
69
|
+
|
|
70
|
+
owner = owner_path(nesting)
|
|
71
|
+
return unless owner
|
|
72
|
+
|
|
73
|
+
separator = nesting.last&.sclass? ? "." : "#"
|
|
74
|
+
node.arguments&.arguments.to_a.each do |argument|
|
|
75
|
+
name = macro_name(argument)
|
|
76
|
+
next unless name
|
|
77
|
+
|
|
78
|
+
generated_names(node.name, name).each do |method_name|
|
|
79
|
+
out << Found.new(id: "#{owner}#{separator}#{method_name}", macro: node.name)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# The methods one macro call generates per named attribute: a reader, a
|
|
85
|
+
# writer, or both for attr_accessor.
|
|
86
|
+
def generated_names(macro, name)
|
|
87
|
+
case macro
|
|
88
|
+
when :attr_reader then [name]
|
|
89
|
+
when :attr_writer then ["#{name}="]
|
|
90
|
+
else [name, "#{name}="]
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def macro_name(argument)
|
|
95
|
+
argument.unescaped if argument.is_a?(Prism::SymbolNode) || argument.is_a?(Prism::StringNode)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Lexical path of the enclosing class/module; `class ::Foo` restarts at
|
|
99
|
+
# the top. A `class << self` scope changes the separator, not the path.
|
|
100
|
+
def owner_path(nesting)
|
|
101
|
+
path = +""
|
|
102
|
+
nesting.each do |scope|
|
|
103
|
+
next if scope.sclass?
|
|
104
|
+
|
|
105
|
+
segment = scope.path
|
|
106
|
+
path.clear if segment.start_with?("::")
|
|
107
|
+
path << "::" unless path.empty?
|
|
108
|
+
path << segment.delete_prefix("::")
|
|
109
|
+
end
|
|
110
|
+
path.empty? ? nil : path
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
data/lib/saboteur/cli/presets.rb
CHANGED
|
@@ -14,6 +14,10 @@ module Saboteur
|
|
|
14
14
|
--html .saboteur/audit-report.html]
|
|
15
15
|
}.freeze
|
|
16
16
|
|
|
17
|
+
# Preset flags that take a separate value token: the pairs a user
|
|
18
|
+
# flag can override in the banner (#666).
|
|
19
|
+
VALUE_FLAGS = %w[--baseline-passes --max-duration --since --operators --json --html].freeze
|
|
20
|
+
|
|
17
21
|
private
|
|
18
22
|
|
|
19
23
|
def preset_or_usage_error(command)
|
|
@@ -31,10 +35,46 @@ module Saboteur
|
|
|
31
35
|
# deprecated inference alias (#245).
|
|
32
36
|
explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
|
|
33
37
|
flags.delete("--all") if explicit_scope
|
|
34
|
-
@err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
|
|
38
|
+
@err.puts("saboteur: preset '#{name}': #{preset_banner(flags).join(' ')} (explicit flags override)")
|
|
35
39
|
@argv = flags + @argv
|
|
36
40
|
run_command
|
|
37
41
|
end
|
|
42
|
+
|
|
43
|
+
# The banner is the reader's only record of the effective settings,
|
|
44
|
+
# so a preset flag the user also passed shows the user's spelling -
|
|
45
|
+
# the later occurrence wins (#666). Flags the user did not override
|
|
46
|
+
# print as the preset declares them.
|
|
47
|
+
def preset_banner(flags)
|
|
48
|
+
overrides = user_flag_overrides
|
|
49
|
+
banner = []
|
|
50
|
+
index = 0
|
|
51
|
+
while index < flags.size
|
|
52
|
+
if VALUE_FLAGS.include?(flags[index])
|
|
53
|
+
banner.concat(overrides.fetch(flags[index], [flags[index], flags[index + 1]]))
|
|
54
|
+
index += 2
|
|
55
|
+
else
|
|
56
|
+
banner << flags[index]
|
|
57
|
+
index += 1
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
banner
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Each long flag the user passed, mapped to its effective spelling:
|
|
64
|
+
# the last occurrence, verbatim, with its separate value token when
|
|
65
|
+
# the flag takes one.
|
|
66
|
+
def user_flag_overrides
|
|
67
|
+
overrides = {}
|
|
68
|
+
@argv.each_with_index do |arg, index|
|
|
69
|
+
name = arg[/\A--[^=]+/] or next
|
|
70
|
+
if arg.include?("=")
|
|
71
|
+
overrides[name] = [arg]
|
|
72
|
+
elsif VALUE_FLAGS.include?(name) && @argv[index + 1]
|
|
73
|
+
overrides[name] = [name, @argv[index + 1]]
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
overrides
|
|
77
|
+
end
|
|
38
78
|
end
|
|
39
79
|
end
|
|
40
80
|
end
|
|
@@ -14,7 +14,9 @@ module Saboteur
|
|
|
14
14
|
set(:seed, v)
|
|
15
15
|
end
|
|
16
16
|
opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
|
|
17
|
-
opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent"
|
|
17
|
+
opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent - " \
|
|
18
|
+
"the clock starts at engine entry, so boot, baselines and the " \
|
|
19
|
+
"coverage map spend it too") do |v|
|
|
18
20
|
set(:max_duration, parse_duration(v))
|
|
19
21
|
end
|
|
20
22
|
opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|
|
|
@@ -18,24 +18,6 @@ module Saboteur
|
|
|
18
18
|
raise UsageError, empty_source_set_message
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
# A --since ref whose resolved range diffs empty yields 0 subjects
|
|
22
|
-
# and a green run over nothing: `saboteur pr <merge_sha>` on a
|
|
23
|
-
# checkout AT the merge diffs the merge against itself (#643).
|
|
24
|
-
# Refuse like a typo'd ref: exit 2, name the base and the range.
|
|
25
|
-
def refuse_empty_since_range
|
|
26
|
-
ref = config["since"]
|
|
27
|
-
return unless ref
|
|
28
|
-
|
|
29
|
-
paths = Selection::GitDiff.changed_paths(ref, root: config.root)
|
|
30
|
-
return unless paths.values.all?(&:empty?)
|
|
31
|
-
|
|
32
|
-
base = Selection::GitDiff.merge_base(ref, root: config.root)
|
|
33
|
-
raise UsageError, "--since #{ref}: the resolved range #{base}..the working tree has no changes - " \
|
|
34
|
-
"the run would pass with 0 subjects. A checkout AT the ref (the CI merge-sha " \
|
|
35
|
-
"checkout) diffs it against itself: fetch the merge's parent, or point --since " \
|
|
36
|
-
"at the base of the change under test"
|
|
37
|
-
end
|
|
38
|
-
|
|
39
21
|
def empty_source_set_message
|
|
40
22
|
include_patterns = config["include"]
|
|
41
23
|
matched = include_patterns.flat_map { |pattern| Dir.glob(pattern, base: config.root) }
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -155,7 +155,6 @@ module Saboteur
|
|
|
155
155
|
return preview if options[:list_operators] || options[:dry_run]
|
|
156
156
|
|
|
157
157
|
refuse_empty_source_set
|
|
158
|
-
refuse_empty_since_range
|
|
159
158
|
execute_run
|
|
160
159
|
rescue ConfigError => e
|
|
161
160
|
@err.puts("saboteur: #{e.message}")
|
|
@@ -393,7 +392,10 @@ module Saboteur
|
|
|
393
392
|
opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
|
|
394
393
|
opts.on("--agent", "Same as --format agent") { @format = "agent" }
|
|
395
394
|
opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
|
|
396
|
-
"a crash prints its backtrace")
|
|
395
|
+
"a crash prints its backtrace") do
|
|
396
|
+
@verbose = true
|
|
397
|
+
set(:verbose, true)
|
|
398
|
+
end
|
|
397
399
|
opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
|
|
398
400
|
@color = v ? :always : :never
|
|
399
401
|
end
|
data/lib/saboteur/config.rb
CHANGED
|
@@ -86,7 +86,10 @@ module Saboteur
|
|
|
86
86
|
"fingerprint_extra" => [],
|
|
87
87
|
"threshold" => nil,
|
|
88
88
|
"max_timeouts" => nil,
|
|
89
|
-
"max_errors" => nil
|
|
89
|
+
"max_errors" => nil,
|
|
90
|
+
# --verbose also lands here, so boot-time output filters (the rspec
|
|
91
|
+
# load-error report, #608) can keep their raw text behind the flag.
|
|
92
|
+
"verbose" => false
|
|
90
93
|
}.freeze
|
|
91
94
|
|
|
92
95
|
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
|
data/lib/saboteur/config_file.rb
CHANGED
|
@@ -9,29 +9,34 @@ module Saboteur
|
|
|
9
9
|
# boots, so it cannot reference app constants - plain Ruby and ENV only.
|
|
10
10
|
module ConfigFile
|
|
11
11
|
# The assignment target. One setter per Config key; anything else is an
|
|
12
|
-
# unknown setting.
|
|
13
|
-
# errors can point at it.
|
|
12
|
+
# unknown setting.
|
|
14
13
|
class Settings
|
|
15
14
|
def initialize
|
|
16
15
|
@values = {}
|
|
17
|
-
@lines = {}
|
|
18
16
|
end
|
|
19
17
|
|
|
20
|
-
attr_reader :values
|
|
18
|
+
attr_reader :values
|
|
21
19
|
|
|
22
|
-
Config::DEFAULTS.
|
|
20
|
+
(Config::DEFAULTS.keys - ["integration"]).each do |key|
|
|
23
21
|
define_method("#{key}=") do |value|
|
|
24
22
|
@values[key] = value
|
|
25
|
-
@lines[key] = caller_locations(1, 1).first.lineno
|
|
26
23
|
end
|
|
27
24
|
end
|
|
28
25
|
|
|
26
|
+
def integration=(value)
|
|
27
|
+
line = caller_locations(1, 1).fetch(0).lineno
|
|
28
|
+
unless value.is_a?(String)
|
|
29
|
+
raise ConfigError, "#{line}: integration must be a String, got #{value.inspect}; use \"rspec\""
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
@values["integration"] = value
|
|
33
|
+
end
|
|
34
|
+
|
|
29
35
|
# Repeatable per-path score gates (#473), the one setting that takes
|
|
30
36
|
# arguments: c.score_gate "app/payments/**/*", min_score: 95. Values
|
|
31
37
|
# are validated later, on the effective Config.
|
|
32
38
|
def score_gate(pattern, min_score:)
|
|
33
39
|
(@values["score_gates"] ||= []) << { "pattern" => pattern, "min_score" => min_score }
|
|
34
|
-
@lines["score_gates"] = caller_locations(1, 1).first.lineno
|
|
35
40
|
end
|
|
36
41
|
|
|
37
42
|
Config::INTEGER_KEYS.each do |key|
|
|
@@ -41,7 +46,6 @@ module Saboteur
|
|
|
41
46
|
raise ConfigError, "#{line}: #{key} must be an Integer, got #{value.inspect}"
|
|
42
47
|
end
|
|
43
48
|
@values[key] = value
|
|
44
|
-
@lines[key] = caller_locations(1, 1).first.lineno
|
|
45
49
|
end
|
|
46
50
|
end
|
|
47
51
|
end
|
data/lib/saboteur/custom_map.rb
CHANGED
|
@@ -64,7 +64,14 @@ module Saboteur
|
|
|
64
64
|
unless spec.is_a?(String)
|
|
65
65
|
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; spec paths must be Strings"
|
|
66
66
|
end
|
|
67
|
-
|
|
67
|
+
|
|
68
|
+
numbered = key.is_a?(Regexp) && spec.match?(/\\\d/)
|
|
69
|
+
named = key.is_a?(Regexp) && spec.match?(/%[-0-9]*[<{][^>}]+[>}]|\\k</)
|
|
70
|
+
if numbered && named
|
|
71
|
+
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
|
|
72
|
+
'regex map specs support \1 backreferences only'
|
|
73
|
+
end
|
|
74
|
+
return if numbered # backreference filled in at match time
|
|
68
75
|
|
|
69
76
|
expanded = File.expand_path(spec, root)
|
|
70
77
|
unless expanded.start_with?("#{root}/")
|
|
@@ -72,6 +79,10 @@ module Saboteur
|
|
|
72
79
|
end
|
|
73
80
|
return if Dir.glob(expanded).any?
|
|
74
81
|
|
|
82
|
+
if named
|
|
83
|
+
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
|
|
84
|
+
'regex map specs support \1 backreferences only'
|
|
85
|
+
end
|
|
75
86
|
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which matches no file"
|
|
76
87
|
end
|
|
77
88
|
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
module Saboteur
|
|
5
|
-
# The warning shown before a run on a tree with uncommitted
|
|
6
|
-
# (#309, #316)
|
|
5
|
+
# The warning shown before a run on a tree with uncommitted
|
|
6
|
+
# changes (#309, #316); the headline says uncommitted or untracked, so a
|
|
7
|
+
# tree of only scratch files is not read as edits (#604). Mutants run your code and can change the tree, so the
|
|
7
8
|
# warning stays; it splits the raw count so a tree full of notes or trial
|
|
8
9
|
# setup files does not read like application edits about to be mutated.
|
|
9
10
|
#
|
|
@@ -101,7 +102,8 @@ module Saboteur
|
|
|
101
102
|
groups = [scope.empty? ? "none inside the mutation scope" : counted("inside the mutation scope", scope),
|
|
102
103
|
counted("outside it", outside)]
|
|
103
104
|
groups << counted("saboteur setup file(s)", setup) unless setup.empty?
|
|
104
|
-
"saboteur: warning: #{@state.changed_files} uncommitted change(s) in the working tree:
|
|
105
|
+
"saboteur: warning: #{@state.changed_files} uncommitted or untracked change(s) in the working tree: " \
|
|
106
|
+
"#{groups.join(', ')}."
|
|
105
107
|
end
|
|
106
108
|
|
|
107
109
|
def counted(label, changes)
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -102,8 +102,12 @@ module Saboteur
|
|
|
102
102
|
@out.puts %(Seatbelt (opt-in, #436): probe #{Execution::Seatbelt.probe} - capability probe only, ) \
|
|
103
103
|
"not verified isolation"
|
|
104
104
|
elsif RUBY_PLATFORM.match?(/darwin/i)
|
|
105
|
+
# Point at the way OUT of the degraded state, like the run
|
|
106
|
+
# warning does (#667): the opt-in Seatbelt sandbox, or silence.
|
|
105
107
|
@out.puts "macOS: mutants run unsandboxed (Landlock is Linux-only) - a container contains " \
|
|
106
108
|
"filesystem writes outside the project"
|
|
109
|
+
@out.puts %(macOS: opt into the sandbox with c.sandbox = "seatbelt" ) \
|
|
110
|
+
"(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
|
|
107
111
|
@out.puts "Network: never restricted on any platform, and a plain container keeps outbound " \
|
|
108
112
|
"access - deny or stub the network when a mutant could reach real APIs"
|
|
109
113
|
end
|