saboteur 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +122 -0
  3. data/README.md +12 -1
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +3 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +29 -0
  19. data/lib/saboteur/cli/preview_command.rb +26 -0
  20. data/lib/saboteur/cli/run_budget_options.rb +3 -0
  21. data/lib/saboteur/cli/run_command.rb +33 -0
  22. data/lib/saboteur/cli/run_output.rb +38 -0
  23. data/lib/saboteur/cli/run_reporting.rb +8 -6
  24. data/lib/saboteur/cli/run_validation.rb +54 -0
  25. data/lib/saboteur/cli.rb +30 -308
  26. data/lib/saboteur/config.rb +72 -6
  27. data/lib/saboteur/constant_subject_finder.rb +3 -0
  28. data/lib/saboteur/custom_map.rb +20 -12
  29. data/lib/saboteur/deep_source.rb +49 -0
  30. data/lib/saboteur/dirty_tree_warning.rb +1 -1
  31. data/lib/saboteur/doctor.rb +6 -0
  32. data/lib/saboteur/engine/baseline_guard.rb +50 -13
  33. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  34. data/lib/saboteur/engine/boot.rb +4 -0
  35. data/lib/saboteur/engine/candidate_selection.rb +15 -5
  36. data/lib/saboteur/engine/candidates.rb +4 -1
  37. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  38. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  39. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  40. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  41. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  42. data/lib/saboteur/engine/coverage_store.rb +278 -26
  43. data/lib/saboteur/engine/diagnostics.rb +23 -6
  44. data/lib/saboteur/engine/distribution.rb +2 -2
  45. data/lib/saboteur/engine/gated_run.rb +1 -0
  46. data/lib/saboteur/engine/load_coverage.rb +54 -8
  47. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  48. data/lib/saboteur/engine/partial_merge.rb +40 -0
  49. data/lib/saboteur/engine/pipelines.rb +7 -7
  50. data/lib/saboteur/engine/reference_specs.rb +71 -0
  51. data/lib/saboteur/engine/result_cache.rb +68 -5
  52. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  53. data/lib/saboteur/engine/run_budget.rb +5 -5
  54. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  55. data/lib/saboteur/engine/sandbox_policy.rb +5 -2
  56. data/lib/saboteur/engine/scheduler.rb +1 -1
  57. data/lib/saboteur/engine/selection_announcement.rb +1 -0
  58. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  59. data/lib/saboteur/engine/shadow_run.rb +38 -0
  60. data/lib/saboteur/engine/subject_selection.rb +33 -37
  61. data/lib/saboteur/engine/suite_partition.rb +22 -0
  62. data/lib/saboteur/engine/test_guess.rb +82 -18
  63. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  64. data/lib/saboteur/engine.rb +41 -9
  65. data/lib/saboteur/environment_hints.rb +55 -0
  66. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  67. data/lib/saboteur/execution/external_runner.rb +105 -1
  68. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  69. data/lib/saboteur/execution/sandbox.rb +30 -17
  70. data/lib/saboteur/execution/supervisor.rb +40 -7
  71. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  72. data/lib/saboteur/execution/worker_pool.rb +48 -4
  73. data/lib/saboteur/frameworks/rails.rb +2 -5
  74. data/lib/saboteur/gates.rb +121 -28
  75. data/lib/saboteur/install.rb +6 -1
  76. data/lib/saboteur/integration/base.rb +19 -2
  77. data/lib/saboteur/integration/rspec.rb +46 -7
  78. data/lib/saboteur/line_hits.rb +81 -0
  79. data/lib/saboteur/method_calls.rb +91 -0
  80. data/lib/saboteur/mutation_source/builtin.rb +34 -22
  81. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  82. data/lib/saboteur/mutation_source/copy_replacement.rb +72 -0
  83. data/lib/saboteur/mutation_source/ensure_removal.rb +49 -0
  84. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  85. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  86. data/lib/saboteur/mutation_source/operator_examples.rb +7 -0
  87. data/lib/saboteur/mutation_source/operator_registry.rb +8 -1
  88. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  89. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  90. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  91. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  92. data/lib/saboteur/preview.rb +9 -1
  93. data/lib/saboteur/refinement_activations.rb +22 -10
  94. data/lib/saboteur/reporter/agent.rb +2 -2
  95. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  96. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  97. data/lib/saboteur/reporter/review_comments.rb +151 -0
  98. data/lib/saboteur/reporter/safe_out.rb +32 -0
  99. data/lib/saboteur/reporter/stryker.rb +8 -6
  100. data/lib/saboteur/reporter/survivor_classification.rb +53 -6
  101. data/lib/saboteur/reporter/survivor_hints.rb +7 -0
  102. data/lib/saboteur/reporter/survivor_list.rb +36 -8
  103. data/lib/saboteur/reporter/terminal.rb +45 -10
  104. data/lib/saboteur/result.rb +14 -4
  105. data/lib/saboteur/selection/coverage_map.rb +136 -10
  106. data/lib/saboteur/selection/external_test_map.rb +4 -0
  107. data/lib/saboteur/selection/git_diff.rb +24 -2
  108. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  109. data/lib/saboteur/selection/recorder.rb +52 -22
  110. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  111. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  112. data/lib/saboteur/subject.rb +16 -0
  113. data/lib/saboteur/subject_finder.rb +91 -6
  114. data/lib/saboteur/text.rb +31 -0
  115. data/lib/saboteur/version.rb +1 -1
  116. data/lib/saboteur/working_tree.rb +7 -2
  117. data/lib/saboteur.rb +8 -0
  118. metadata +46 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0a4dcacd09e6b4ca97537446ae6cbc73b13fb2c6dabcc26f22ea26784e869151
4
- data.tar.gz: 83c7e9f8121a1193c518cbfcba50b825d13b0e08ccd90f3085f88d932855fd03
3
+ metadata.gz: e7d127c4a50466eb11e40c8f1f8ac92b013621d76ddfd3f122823ec85f0ff726
4
+ data.tar.gz: 8768ae6add39713a3ffb0222609c096ddb8e71e7edcb174f83e0ce3719c6296f
5
5
  SHA512:
6
- metadata.gz: a9baa6f9af8e3f2ecd0e2fd661104550e23b7b5e2f368d4cd996b5648994537888d88ba1130477525b93d4df13a756585eed3f9ffb27f309057e7452d069a982
7
- data.tar.gz: 782690794b535288178addb949cf7b0e530e19d4e55acd20c837903f4ddb8818581b42cacfc5af1d1ccb7ba4a6dd9e0cbd84a5059f143407f99195e2ed14a046
6
+ metadata.gz: eb07cd559cf3b12dadd0e7b1a148f04e2e6aada3e961f9dfcd249f05da69cd7f59bc8a43335450df86ae4b6406be013ca7e737357ac3e07023169f6f9d434e52
7
+ data.tar.gz: 1cb27f6614903b63907f1ddf54872bdcfa1e1ab2c288bd40f4ed3ce9e247bed320105afb33fcb03e845a03887f7343250c36d2b82f35a1c7da61227dd83f0244
data/CHANGELOG.md CHANGED
@@ -6,6 +6,128 @@ All notable changes are documented here. The format follows
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.10.0] - 2026-10-21
10
+
11
+ ### Added
12
+
13
+ - Survivor labels in the terminal: surviving mutants carry a label and
14
+ the findings collapse per statement, ignored mutants are reported with
15
+ their reasons, and external-runner runs emit the `tool_limit` label
16
+ (#1065, #1071).
17
+ - Pins covering the terminal collapse, the `--all-mutants` wiring, and
18
+ the phase1 end-to-end default (#1065).
19
+
20
+ ### Changed
21
+
22
+ - The terminal disambiguates the two no-coverage skip counts (#1087).
23
+ - `test_selection = :all` is gone: the 17 mirror-breaking files are
24
+ mapped explicitly instead (#1077).
25
+
26
+ ### Fixed
27
+
28
+ - The `element_removal` help text shows that any array literal works
29
+ (#1086).
30
+
31
+ ## [0.9.0] - 2026-10-20
32
+
33
+ ### Added
34
+
35
+ - Five new mutation operators: `and_assign` drops the guard on `&&=`
36
+ (#873), `jump_value` replaces unchecked early-exit values (#875),
37
+ `rescue_removal` removes exception handlers no test reaches (#877),
38
+ `when_removal` removes case branches no test takes (#887), and
39
+ `unary_removal` removes signs and complements no test sees (#890).
40
+ - `--resume DIR` keeps verdicts in a directory across runs, with a
41
+ manifest check on the directory (#840, #831), and `--incremental`
42
+ checkpoints verdicts while the run is going (#820, documented in #822).
43
+ - Partial and parallel coverage: `c.coverage_parts` deals the suite into
44
+ parts built in parallel (#741: #769, #772, #774, #776), keeps each
45
+ finished part when a build stops (#860), and reuses the saved parts
46
+ when a parallel build is rerun (#862). The partial map merge and an
47
+ opt-in shadow run compare a partial rebuild against the full map
48
+ (#829, #832), and `partial_coverage = "use"` skips the full build when
49
+ only spec files or covered sources changed, rerunning just the stale
50
+ spec files and merging them into the earlier map (#918).
51
+ - TracePoint coverage recorders: `MethodCalls` records Ruby method
52
+ entries (#794, #795), a line-hits recorder reads branches once per
53
+ spec file (#894, #897), and `c.coverage_recorder = "trace"` picks the
54
+ TracePoint build (#898).
55
+ - Constant-reference spec selection: a spec that requires a spec-less
56
+ file counts as its reference reader (#876, #879), a stop that keeps a
57
+ hub file's cap says how many specs reference it (#881), and the agent
58
+ JSON provenance reports and flags findings whose specs were chosen by
59
+ reference (#884, #885).
60
+ - The GitHub Action posts surviving mutants as pull-request review
61
+ comments on the lines the PR head has (#788, #789, #793), matches the
62
+ review comment bot login without regard to case (#791), documents the
63
+ review comment inputs (#790), and exposes max-timeouts and max-errors
64
+ as inputs (#246, #767).
65
+ - Integration targets: a target can name its own gemfile (#900), carry
66
+ patch, env and recorder options (#899), and run setup commands and add
67
+ gems (#910). doorkeeper, paper_trail, cancancan and lobsters join the
68
+ recorder-parity targets (#901, #908, #912).
69
+ - The JSON report adds per-gate score breakdowns (#473, #766) and a
70
+ `saboteur.complete` field (#784).
71
+ - A real-bug corpus runner with a manual CI workflow, and the s-z and
72
+ i-r corpus slices (#652: #915, #916).
73
+ - A coverage cache miss says why it missed and what changed (#812), and
74
+ spec shards balance by recorded per-example run time (#747, #748).
75
+ - Warnings across the survivor, sandbox grant, shadow, worker, budget
76
+ and `--since`/`--subject` messages lead with the outcome and name the
77
+ remedy, per the documented message style (#332 series, #796).
78
+
79
+ ### Fixed
80
+
81
+ - `--since` ignores a line whose only change is its trailing comment
82
+ (#892, fixed in #921).
83
+ - `script/rspec-shard` no longer dies of SIGPIPE on a long spec list
84
+ (#878).
85
+ - `or_assign` no longer mutates constant targets inside a def (#874).
86
+ - A test that suspends Coverage no longer breaks the map build; Coverage
87
+ resumes after it (#869).
88
+ - Worker pool cleanup no longer leaks live entries or masks errors, and
89
+ keeps unregistering workers when the orphan sweep goes blind (#762:
90
+ #770, #771); containment and the live registry survive a blind process
91
+ table (#762: #764).
92
+ - A timed-out Gemfile resolver no longer blocks on escaped-grandchild
93
+ pipes, and the resolver-pipe specs reap escaped grandchildren (#734,
94
+ #750). A `path:` source equal to the project root rebases onto the
95
+ copy root (#729), external test-command `path:` sources are refused or
96
+ rebased (#644, #721), and the BUNDLE_GEMFILE pin gaps close with
97
+ lockfile-mismatch refusals (#732).
98
+ - A cached coverage map is read as untrusted input: its field types,
99
+ durations and loaded files are checked, non-finite values are
100
+ rejected, and the inputs snapshot binds to the map and the runtime
101
+ (#853, #854, #859, #825). Stored verdicts are type-checked before
102
+ reuse (#842).
103
+ - Cache files and the coverage map write through an exclusive private
104
+ temp file (#846, #850).
105
+ - Registry DSL blocks in Ohai/Puppet style promote their defs to
106
+ subjects (#700, #727).
107
+ - The baseline checks the unchanged suite in the coverage build's parts
108
+ instead of a separate pass (#930).
109
+
110
+ ### Security
111
+
112
+ - CI workflow-command injection is closed at every sink: the unmatched
113
+ whole_file pattern (#856), repo-controlled file names in the
114
+ constant-safety warnings (#857), name_list items (#858),
115
+ repo-controlled names in the syntax, custom map and sandbox grant
116
+ warnings (#870), every Diagnostics warning line (#871), the terminal
117
+ report at one choke point (#872), the stop and gate messages (#919),
118
+ and the CLI error sinks (#917, fixed in #920).
119
+ - The review comment planner is hardened against untrusted input (#783),
120
+ and the Action fetches the pull request base with the token in a step
121
+ of its own, with the token-handling scripts pinned (#797, #798).
122
+
123
+ ### Documentation
124
+
125
+ - The getting-started guide shows a gem-installed run (#815), the
126
+ config-key table documents every key and the Rails config points at it
127
+ (#809), the Action guide documents a per-path score gate (#817),
128
+ `--incremental` verdict saving is documented (#822), and the warning
129
+ and note style convention is written down (#796).
130
+
9
131
  ## [0.8.1] - 2026-10-18
10
132
 
11
133
  ### Added
data/README.md CHANGED
@@ -89,7 +89,8 @@ killed 8 survived 2 timeout 0 resource limit 0 error 0 score 80.0 %
89
89
  Each survivor comes with its diff, the tests that ran, and a triage label
90
90
  (`likely_equivalent`, `arid`, `missing_test`) to sort what to read first.
91
91
  The labels are heuristics for reading order, never a diagnosis, and they
92
- never change the score.
92
+ never change the score. The terminal prints one finding per statement with
93
+ the label mix and mutant count; `--all-mutants` lists every mutant.
93
94
 
94
95
  A run on a tree with uncommitted changes opens with a warning that splits
95
96
  the count into changes inside the mutation scope, outside it, and
@@ -127,6 +128,16 @@ the macro itself `ran at load` instead of letting its mutants survive.
127
128
  before a kill is rejected, so its mutants are still scored.
128
129
  - **Reports for people and tools.** Terminal, HTML, a Stryker-compatible
129
130
  JSON report, and a stable agent JSON API with a deprecation policy.
131
+ The GitHub Action posts surviving mutants as review comments on the
132
+ diff lines of the pull request.
133
+ - **Resumable runs.** `--resume DIR` keeps verdicts across runs and
134
+ `--incremental` checkpoints them while the run is going, so a stopped
135
+ run picks up where it left off.
136
+ - **Faster coverage builds.** `c.coverage_parts` builds the coverage map
137
+ in parallel parts, `partial_coverage = "use"` rebuilds only the stale
138
+ spec files and merges them into the saved map, and
139
+ `c.coverage_recorder = "trace"` builds the map with a TracePoint
140
+ recorder.
130
141
  - **A plan before the run.** `saboteur run --dry-run --format json` prints
131
142
  the versioned [mutant inventory](docs/dry-run-json.md) - every planned
132
143
  mutant with its stable digest and edits - without booting the suite.
data/exe/saboteur CHANGED
@@ -11,7 +11,7 @@
11
11
  # (one line, invalid bytes scrubbed, bounded length) still exits 3.
12
12
  begin
13
13
  require "coverage"
14
- Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
14
+ Coverage.setup(lines: true, branches: true) if Coverage.state == :idle
15
15
 
16
16
  require "saboteur/crash_report"
17
17
  rescue LoadError, StandardError => e
@@ -27,7 +27,11 @@ end
27
27
  # ones. The engine suspends counting right after boot and reads the
28
28
  # counters as the map's load lines (a forked child would not inherit them).
29
29
  begin
30
- Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
30
+ if ARGV.first == "run" && Coverage.state == :suspended
31
+ require "saboteur/method_calls"
32
+ Saboteur::MethodCalls.boot
33
+ Coverage.resume
34
+ end
31
35
 
32
36
  require "saboteur/cli"
33
37
 
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "prism"
5
+ require_relative "deep_source"
5
6
  require_relative "subject"
6
7
 
7
8
  module Saboteur
@@ -24,6 +25,8 @@ module Saboteur
24
25
  result = Prism.parse(File.binread(file), filepath: file)
25
26
  return [] if result.failure?
26
27
 
28
+ DeepSource.check!(result.value, file)
29
+
27
30
  found = []
28
31
  walk(result.value, [], found)
29
32
  found
@@ -3,6 +3,7 @@
3
3
 
4
4
  require_relative "identity"
5
5
  require_relative "completeness"
6
+ require_relative "../config"
6
7
  require_relative "../engine/run_fingerprint"
7
8
 
8
9
  module Saboteur
@@ -11,7 +12,7 @@ module Saboteur
11
12
  # This does not attest external database, network or environment state.
12
13
  class RunEvidence
13
14
  SAFE_DROPS = %w[unchanged duplicate].freeze
14
- OUTPUT_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
15
+ OUTPUT_SETTINGS = Config::OUTPUT_KEYS
15
16
 
16
17
  def initialize(config:, extra_files: [])
17
18
  @config = config
@@ -0,0 +1,59 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Agent-mode provenance: the record a JSON report carries so its rerun
7
+ # and replay commands reproduce the run exactly (#334). Extracted from
8
+ # CLI; the builder and its settings helpers are private to the mixin.
9
+ module AgentProvenance
10
+ private
11
+
12
+ def agent_provenance(report, tree, engine)
13
+ Reporter::Agent::Provenance.new(
14
+ ref: config["since"], seed: report.seed, head: tree.head, working_tree: tree.label,
15
+ changed_files: tree.changed_files, mode: agent_mode, json_report: config.path("json_report"),
16
+ subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
17
+ sources: @sources || [],
18
+ config_path: options[:config], digest: options[:id], cache: config["cache"],
19
+ incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings,
20
+ referenced: report.referenced
21
+ )
22
+ end
23
+
24
+ # The settings keys typed on the command line: a typed empty operator
25
+ # list may clear a config value, so the generated commands keep it.
26
+ def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
27
+
28
+ # A whole-suite run records its scope as --all, so the generated rerun
29
+ # and replay commands do not default back to inference (which can exit
30
+ # 2 in a project with a spec-less source file).
31
+ def agent_settings
32
+ settings = options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
33
+ .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
34
+ # The effective mode: named tests win over a configured whole-suite
35
+ # selection by precedence, so all is recorded only when no tests
36
+ # resolved - otherwise the follow-up commands would carry both --all
37
+ # and --test, which the CLI rejects.
38
+ settings[:all] = true if (config["all"] || config["test_selection"] == "all") && config["tests"].empty?
39
+ # CLI-only switches that change candidate availability or boot, so a
40
+ # printed rerun/replay must keep them (config-file settings like
41
+ # arid/hazard patterns, whole-file, require, rspec_args and rails ride
42
+ # along through --config or the default config path).
43
+ settings[:allow_hazardous] = true if options[:allow_hazardous]
44
+ settings[:allow_focus] = true if options[:allow_focus]
45
+ # An explicit false (--no-max-duration-ok over a config true) is a
46
+ # setting too: dropping it would silently re-enable the config value.
47
+ settings[:max_duration_ok] = options[:max_duration_ok] unless options[:max_duration_ok].nil?
48
+ settings
49
+ end
50
+
51
+ def agent_mode
52
+ return "replay" if options[:id]
53
+ return "diff" if config["since"]
54
+
55
+ @sources&.any? ? "files" : "full"
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,28 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The uncommitted-work warning printed before a run (#334). Extracted
7
+ # from CLI.
8
+ module DirtyTreeNotice
9
+ private
10
+
11
+ # Mutants run your code, and a mutant of code that writes or deletes files
12
+ # can change the tree. Say so up front when uncommitted work is at stake.
13
+ # The scope the warning names is the configured one: positional source
14
+ # operands narrow the run after the warning, so the include/exclude they
15
+ # imply stay out of its config (#309, #503 review).
16
+ def warn_dirty_tree
17
+ warning = if @sources
18
+ Config.load(path: options[:config],
19
+ overrides: overrides.except(:include,
20
+ :exclude))
21
+ else
22
+ config
23
+ end
24
+ DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,41 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # `saboteur doctor`: environment report and optional unchanged suite
7
+ # passes, never mutations (#334). Extracted from CLI by command.
8
+ module DoctorCommand
9
+ def doctor_parser
10
+ OptionParser.new do |opts|
11
+ opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N] [--suite] [--test PATH]"
12
+ opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |value| set(:config, value) }
13
+ opts.on("--seed N", Integer, "Seed for the two baseline runs (default 1)") { |value| set(:seed, value) }
14
+ opts.on("--suite", "Also run the unchanged suite twice (slow on a large suite)") { set(:suite, true) }
15
+ opts.on("--test PATH", "Spec file or directory for the suite runs; repeatable. Implies --suite") do |value|
16
+ (options[:tests] ||= []) << value
17
+ end
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def doctor_command = help_requested? ? show_help("doctor") : doctor
24
+
25
+ def doctor
26
+ doctor_parser.parse!(@argv)
27
+ raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
28
+
29
+ options[:tests]&.each { |test| require_existing_test(test) }
30
+ Doctor.new(config, out: @out, suite: options[:suite] || options.key?(:tests)).call
31
+ EXIT_OK
32
+ rescue ConfigError => e
33
+ @err.puts("saboteur: #{Text.display_safe_lines(e.message)}")
34
+ EXIT_USAGE
35
+ rescue Error => e
36
+ @err.puts("saboteur: #{Text.display_safe_lines(e.message)}")
37
+ EXIT_TOOL
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,40 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The detail line a "mostly errored" or "incomplete" exit message adds:
7
+ # the most common first line of the errored results' reasons, so a run
8
+ # that exits on its error count still names why (a worker killed by a
9
+ # signal, say) instead of leaving the cause in the stdout error list.
10
+ module ErrorCause
11
+ def self.incomplete(problem)
12
+ "saboteur: #{problem.size} mutant(s) errored or hit a resource limit, so the score is " \
13
+ "incomplete#{line(problem)}"
14
+ end
15
+
16
+ LIMIT = 200
17
+
18
+ # "" when no result carries a reason.
19
+ def self.line(errored)
20
+ reasons = errored.filter_map { |result| first_line(result.reason) }.reject(&:empty?)
21
+ return "" if reasons.empty?
22
+
23
+ # Worker numbers and pids differ between otherwise identical deaths.
24
+ group = reasons.group_by { |reason| reason.delete("0-9") }.values.max_by(&:size)
25
+ "\n most common cause (#{group.size} of #{errored.size}): #{group.first}"
26
+ end
27
+
28
+ # A reason can carry invalid bytes, terminal escapes or a huge line,
29
+ # and this runs on the way to the exit code: scrub, escape controls and
30
+ # cap before anything groups or prints it, and never raise.
31
+ def self.first_line(reason)
32
+ text = Text.utf8(reason.to_s).lines.map(&:strip).find { |line| !line.empty? }.to_s
33
+ text = Text.display_safe(text.gsub(/\e\[[0-9;]*[A-Za-z]/, ""))
34
+ text.length > LIMIT ? "#{text[0, LIMIT]}..." : text
35
+ rescue StandardError
36
+ reason.to_s.dump[0, LIMIT]
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,46 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Execution, cache and hazard flags for `saboteur run` (#334).
7
+ module ExecutionOptions
8
+ private
9
+
10
+ def define_execution_options(opts)
11
+ opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
12
+ opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
13
+ add_run_budget_options(opts)
14
+ opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
15
+ opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
16
+ opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
17
+ opts.on("--no-cache", "Rebuild the coverage map") { set(:cache, false) }
18
+ opts.on("--allow-hazardous",
19
+ "Also run mutants inside destructive calls (rm_rf, system, ...)") { set(:allow_hazardous, true) }
20
+ opts.on("--fail-fast",
21
+ "Stop after the first subject with a survivor (needs --jobs 1)") { set(:fail_fast, true) }
22
+ add_reuse_options(opts)
23
+ opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
24
+ set(:allow_focus, true)
25
+ end
26
+ opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
27
+ "mutated files") { set(:all, true) }
28
+ opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
29
+ "For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
30
+ set(:test_command, v)
31
+ end
32
+ opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
33
+ "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
34
+ set(:guess_tests, v)
35
+ end
36
+ end
37
+
38
+ def add_reuse_options(opts)
39
+ opts.on("--incremental", "Reuse earlier verdicts whose inputs did not change") { set(:incremental, true) }
40
+ opts.on("--resume DIR", "Keep verdicts in DIR and reuse them in a later run with the same inputs") do |v|
41
+ set(:resume, v)
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -14,11 +14,9 @@ module Saboteur
14
14
  end
15
15
 
16
16
  # The crash log lives with the other run state. A broken config (or a
17
- # nil cache_dir) must not swallow the crash report itself, so fall
18
- # back to no log.
17
+ # nil cache_dir) must not swallow the crash report itself: no log.
19
18
  def crash_log_path
20
- dir = config.path("cache_dir")
21
- dir && File.join(dir, CrashReport::LOG_NAME)
19
+ config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
22
20
  rescue StandardError
23
21
  nil
24
22
  end
@@ -34,7 +32,8 @@ module Saboteur
34
32
 
35
33
  return EXIT_TOOL if incomplete_under_threshold?(report)
36
34
 
37
- failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
35
+ quality_failed = failed_quality_gate?(report)
36
+ gates.failed?(report, @err, record_cause: !quality_failed) || quality_failed ? EXIT_SURVIVORS : EXIT_OK
38
37
  end
39
38
 
40
39
  # A spent --max-duration budget exits 4 on its own, before the gates:
@@ -62,7 +61,7 @@ module Saboteur
62
61
  def unverifiable_scope?(report)
63
62
  if unverifiable_review?(report)
64
63
  report.skipped_subjects.each do |subject, reason|
65
- @err.puts("saboteur: #{subject.id} was skipped (#{reason}), so the change under test was not verified")
64
+ @err.puts(subject.skipped_gate_sentence(reason, report.root))
66
65
  end
67
66
  return true
68
67
  end
@@ -86,10 +85,9 @@ module Saboteur
86
85
  def mostly_errored?(report)
87
86
  return false unless report.majority_errored?
88
87
 
89
- executed = report.executed
90
- errors = executed.count { |result| result.status == :error }
91
- @err.puts("saboteur: #{errors} of #{executed.size} mutants errored, so the run tested almost nothing; " \
92
- "fix the errors and run again")
88
+ errored = report.executed.select { |result| result.status == :error }
89
+ @err.puts("saboteur: fix the errors and run again\n #{errored.size} of #{report.executed.size} " \
90
+ "mutants errored, so the run tested almost nothing#{ErrorCause.line(errored)}")
93
91
  true
94
92
  end
95
93
 
@@ -159,10 +157,10 @@ module Saboteur
159
157
  def incomplete_under_threshold?(report)
160
158
  return false if config["threshold"].nil?
161
159
 
162
- count = report.counts.values_at(*(gates.judges_errors? ? %i[resource_limit] : %i[error resource_limit])).sum
163
- @err.puts("saboteur: #{count} mutant(s) errored or hit a resource limit, so the score is incomplete") if
164
- count.positive?
165
- count.positive?
160
+ statuses = gates.judges_errors? ? %i[resource_limit] : %i[error resource_limit]
161
+ problem = report.results.select { |result| statuses.include?(result.status) }
162
+ @err.puts(ErrorCause.incomplete(problem)) if problem.any?
163
+ problem.any?
166
164
  end
167
165
  end
168
166
  end
@@ -0,0 +1,21 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Explicit test selection and quality-gate flags (#334).
7
+ module ExplicitFileOptions
8
+ private
9
+
10
+ def define_explicit_file_options(opts)
11
+ opts.on("--test PATH", "Spec file or directory to run; repeatable. Replaces the configured spec paths") do |v|
12
+ (options[:tests] ||= []) << v
13
+ end
14
+ opts.on("--threshold PERCENT", Float, "Exit 1 when the score is below PERCENT (0-100)") do |v|
15
+ set(:threshold, v)
16
+ end
17
+ Gates.define_options(opts) { |key, value| set(key, value) }
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,46 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The source and test operands of `saboteur run`, checked against the
7
+ # file system before anything boots (#334). Extracted from CLI.
8
+ module ExplicitSources
9
+ private
10
+
11
+ # Positional arguments name the files (or directories) to mutate. They
12
+ # replace the configured include and exclude patterns. Each must exist,
13
+ # so a typo is an error and not a run over zero subjects.
14
+ def use_explicit_sources
15
+ Selection::SourceScope.verify_no_specs!(@argv)
16
+ options[:tests]&.each { |test| require_existing_test(test) }
17
+ # --test may stand alone: the configured include (or the --since
18
+ # diff) names the sources. The files must still exist.
19
+ return if @argv.empty?
20
+
21
+ @sources = @argv.dup
22
+ options[:include] = @argv.map { |source| source_pattern(source) }
23
+ options[:exclude] = []
24
+ end
25
+
26
+ def source_pattern(source)
27
+ return "#{glob_escape(source.chomp('/'))}/**/*.rb" if File.directory?(source)
28
+ raise UsageError, "source is not a Ruby file: #{source}" if File.file?(source) && !source.end_with?(".rb")
29
+ raise UsageError, "source file not found: #{source}" unless File.file?(source)
30
+
31
+ glob_escape(source)
32
+ end
33
+
34
+ # include is a list of glob patterns; a named file must match only itself.
35
+ def glob_escape(path)
36
+ path.gsub(/[\\*?\[\]{}]/) { |character| "\\#{character}" }
37
+ end
38
+
39
+ # A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
40
+ def require_existing_test(test)
41
+ path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
42
+ raise UsageError, "test file not found: #{test}" unless File.exist?(path)
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,35 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Where a failed command speaks: stderr for people, and in agent mode the
7
+ # same failure as JSON on stdout (#334). Extracted from CLI.
8
+ module FailureOutput
9
+ private
10
+
11
+ def usage_error(message = nil)
12
+ @err.puts(message) if message
13
+ @err.puts(USAGE)
14
+ agent_failure(message || USAGE, EXIT_USAGE)
15
+ end
16
+
17
+ def agent? = @format == "agent"
18
+
19
+ # In agent mode the failure is also on stdout, where the agent reads its answer.
20
+ def agent_failure(message, exit_code)
21
+ @out.print(Reporter::Agent.render(Reporter::Agent.failure(message, exit_code))) if agent?
22
+ exit_code
23
+ end
24
+
25
+ # Error messages interpolate repository-controlled names, so each line is
26
+ # made safe before it is printed (a newline then "::error::" in a file
27
+ # name must not become a workflow command in the CI log).
28
+ def report_failure(error, exit_code)
29
+ message = Text.display_safe_lines(error.message)
30
+ @err.puts("saboteur: #{message}")
31
+ agent_failure(message, exit_code)
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,23 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # `saboteur help` and the `--help` of every command (#334). Extracted
7
+ # from CLI by command.
8
+ module HelpCommand
9
+ private
10
+
11
+ def help_requested? = @argv.intersect?(%w[--help -h])
12
+
13
+ def show_help(command_name)
14
+ help = Help.new(option_lines: option_lines)
15
+ text = command_name ? help.command(command_name) : help.overview
16
+ return @out.puts(text) || EXIT_OK if text
17
+
18
+ @err.puts("unknown command: #{command_name}. Commands: #{help.command_names.join(', ')}")
19
+ EXIT_USAGE
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,17 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The --matrix flag for `saboteur run` (#334).
7
+ module MatrixOption
8
+ private
9
+
10
+ def define_matrix_option(opts)
11
+ opts.on("--matrix", "Run every covering test per mutant and list blind and redundant tests") do
12
+ set(:matrix, true)
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end