saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
data/lib/saboteur/gates.rb
CHANGED
|
@@ -41,12 +41,18 @@ module Saboteur
|
|
|
41
41
|
|
|
42
42
|
def initialize(config)
|
|
43
43
|
@limits = SETTINGS.keys.to_h { |key| [key, self.class.limit(config[key], key)] }
|
|
44
|
+
@score_gates = Array(config["score_gates"])
|
|
44
45
|
end
|
|
45
46
|
|
|
46
47
|
attr_reader :cause
|
|
47
48
|
|
|
48
49
|
def any?
|
|
49
|
-
@limits.values.any?
|
|
50
|
+
@limits.values.any? || @score_gates.any?
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# The flag-style name of the configured gate, for the --agent rejection.
|
|
54
|
+
def agent_option_name
|
|
55
|
+
@score_gates.any? ? "score gates" : "--max-* gates"
|
|
50
56
|
end
|
|
51
57
|
|
|
52
58
|
# --max-errors judges errored mutants, so a threshold run tolerates them.
|
|
@@ -56,6 +62,7 @@ module Saboteur
|
|
|
56
62
|
|
|
57
63
|
# Prints each failed gate to err and remembers the first one's cause for the JSON report.
|
|
58
64
|
def failed?(report, err)
|
|
65
|
+
vacuous_gates(report).each { |note| err.puts("saboteur: #{note}") }
|
|
59
66
|
found = violations(report)
|
|
60
67
|
found.each { |violation| err.puts("saboteur: #{violation.message}") }
|
|
61
68
|
@cause = found.first&.cause
|
|
@@ -69,11 +76,45 @@ module Saboteur
|
|
|
69
76
|
next if limit.nil? || !over?(count, limit, report.executed.size)
|
|
70
77
|
|
|
71
78
|
Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
|
|
72
|
-
end
|
|
79
|
+
end + score_gate_violations(report)
|
|
73
80
|
end
|
|
74
81
|
|
|
75
82
|
private
|
|
76
83
|
|
|
84
|
+
# Per-path gates (#473): each gate scores the results whose subject file
|
|
85
|
+
# (root-relative) matches its pattern, with the run's own Score formula,
|
|
86
|
+
# and fails under its floor.
|
|
87
|
+
def score_gate_violations(report)
|
|
88
|
+
@score_gates.filter_map do |gate|
|
|
89
|
+
scored = score_gate_subset(gate, report)
|
|
90
|
+
breakdown = Score.of(scored)
|
|
91
|
+
next if breakdown.tested.zero?
|
|
92
|
+
|
|
93
|
+
score = breakdown.value * 100
|
|
94
|
+
next if score >= gate["min_score"]
|
|
95
|
+
|
|
96
|
+
Violation.new(cause: "scoreGate",
|
|
97
|
+
message: "score gate #{gate['pattern']}: score #{format('%.1f', score)} below " \
|
|
98
|
+
"#{gate['min_score']} (#{scored.size} mutants under it)")
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# A gate with nothing executed under it passes vacuously and says so.
|
|
103
|
+
def vacuous_gates(report)
|
|
104
|
+
@score_gates.filter_map do |gate|
|
|
105
|
+
breakdown = Score.of(score_gate_subset(gate, report))
|
|
106
|
+
"score gate #{gate['pattern']}: nothing executed under it, passing vacuously" if breakdown.tested.zero?
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def score_gate_subset(gate, report)
|
|
111
|
+
prefix = File.join(File.realpath(report.root), "")
|
|
112
|
+
report.results.select do |result|
|
|
113
|
+
relative = result.subject.file.delete_prefix(prefix)
|
|
114
|
+
File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
77
118
|
def over?(count, limit, executed)
|
|
78
119
|
return count > limit if limit.is_a?(Integer)
|
|
79
120
|
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -25,7 +25,12 @@ module Saboteur
|
|
|
25
25
|
2 => "Usage or configuration error: fix the command or config/saboteur.rb. A replay --id that matches no " \
|
|
26
26
|
"mutant is only found after the target has booted. A named source or test file that does not exist.",
|
|
27
27
|
3 => "Tool error: saboteur could not finish, or more than half of the executed mutants errored (such " \
|
|
28
|
-
"a run tested almost nothing)
|
|
28
|
+
"a run tested almost nothing), or a --since run skipped every subject of the change (for example " \
|
|
29
|
+
"refinements or no coverage), so the change is unverified. With --threshold, also any errored or " \
|
|
30
|
+
"resource-limited mutant.",
|
|
31
|
+
4 => "Time budget spent: --max-duration stopped the run before the next subject, so the score is " \
|
|
32
|
+
"partial. --max-duration-ok leaves the exit to the plain verdict: 0, 1, or 3 when the budget " \
|
|
33
|
+
"stopped the run before any mutant was tested."
|
|
29
34
|
}.freeze
|
|
30
35
|
|
|
31
36
|
DOCTOR_EXIT_CODE_MEANINGS = {
|
|
@@ -40,16 +45,22 @@ module Saboteur
|
|
|
40
45
|
summary: "Mutate your code and report the mutants your tests do not catch",
|
|
41
46
|
description: "Builds a per-test coverage map, makes small changes to your methods, runs only the " \
|
|
42
47
|
"tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
|
|
43
|
-
"directories after run replace include and exclude
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
48
|
+
"directories after run replace include and exclude. Without --test, the run infers the " \
|
|
49
|
+
"specs named after the mutated files and runs only those (with --since, plus every " \
|
|
50
|
+
"spec file the diff adds or changes that the suite loads); a mutated file with no " \
|
|
51
|
+
"conventional spec stops the run with exit 2 before anything boots, and --all keeps " \
|
|
52
|
+
"the whole configured suite. --threshold PERCENT replaces the survivor check: exit 0 " \
|
|
53
|
+
"when the score is at or above PERCENT, exit 1 when it is below. All operators run " \
|
|
54
|
+
"unless you select a smaller set; exclusions always remove operators from that set. " \
|
|
55
|
+
"Inline comment syntax: " \
|
|
47
56
|
'# saboteur:disable-line [operator ...] reason="...". A reason is required.',
|
|
48
57
|
takes_options: true,
|
|
49
58
|
exit_codes: EXIT_CODE_MEANINGS,
|
|
50
59
|
examples: [
|
|
51
|
-
["Pull request: only methods changed since the target branch
|
|
52
|
-
|
|
60
|
+
["Pull request: only methods changed since the target branch, with the inferred specs",
|
|
61
|
+
"saboteur run --since origin/main"],
|
|
62
|
+
["Whole project and whole suite, four workers, repeatable seed", "saboteur run --all --jobs 4 --seed 42"],
|
|
63
|
+
["One deterministic CI shard of four", "saboteur run --all --shard 1/4"],
|
|
53
64
|
["One class or method", "saboteur run --subject 'Person#adult?'"],
|
|
54
65
|
["Replay one mutant by the digest printed next to a survivor", "saboteur run --id 1a2b3c4d5e6f7a8b"],
|
|
55
66
|
["Mutate named files against named specs, skipping the whole-suite coverage run",
|
|
@@ -58,12 +69,41 @@ module Saboteur
|
|
|
58
69
|
["AI agent: bounded JSON summary and strict exit codes", "saboteur run --agent --since origin/main"]
|
|
59
70
|
]
|
|
60
71
|
),
|
|
72
|
+
Command.new(
|
|
73
|
+
name: "pr",
|
|
74
|
+
summary: "Preset: the fast pull-request loop over the lines changed since a ref",
|
|
75
|
+
description: "Expands to saboteur run --baseline-passes 1 --max-duration 10m " \
|
|
76
|
+
"--since REF, with REF the first operand (default main): one baseline check, a " \
|
|
77
|
+
"ten-minute wall-clock budget, and mutants limited to the changed methods. The " \
|
|
78
|
+
"preset fails with exit 4 when the budget is spent; the summary counts the " \
|
|
79
|
+
"unrun mutants. Add --max-duration-ok to accept a partial verdict; other " \
|
|
80
|
+
"gates still apply. Every run option still works; explicit flags win (#654).",
|
|
81
|
+
takes_options: true,
|
|
82
|
+
exit_codes: EXIT_CODE_MEANINGS,
|
|
83
|
+
examples: [["Changed lines since main", "saboteur pr"],
|
|
84
|
+
["Changed lines since another target", "saboteur pr origin/develop"],
|
|
85
|
+
["The same loop with two baseline checks", "saboteur pr --baseline-passes 2"]]
|
|
86
|
+
),
|
|
87
|
+
Command.new(
|
|
88
|
+
name: "audit",
|
|
89
|
+
summary: "Preset: the whole project, all operators, both reports",
|
|
90
|
+
description: "Expands to saboteur run --all --operators all --baseline-passes 2 " \
|
|
91
|
+
"--json .saboteur/audit-report.json --html .saboteur/audit-report.html: every " \
|
|
92
|
+
"subject against the whole suite, all operators, both baseline checks, writing " \
|
|
93
|
+
"the Stryker JSON and HTML reports. Configured selection limits (since, " \
|
|
94
|
+
"subjects, exclude_operators, max_duration) still apply; an explicit flag after " \
|
|
95
|
+
"the preset wins (#244), and an explicit --test replaces the injected --all.",
|
|
96
|
+
takes_options: true,
|
|
97
|
+
exit_codes: EXIT_CODE_MEANINGS,
|
|
98
|
+
examples: [["Full project audit with JSON and HTML reports", "saboteur audit"]]
|
|
99
|
+
),
|
|
61
100
|
Command.new(
|
|
62
101
|
name: "doctor",
|
|
63
102
|
summary: "Check the environment, and with --suite run the baseline twice, without mutating anything",
|
|
64
103
|
description: "Boots the target like a run and checks runtime, coverage and boot files. With --suite " \
|
|
65
104
|
"it also runs your suite twice in the same order with one seed and reports whether both " \
|
|
66
|
-
"runs passed; --test PATH limits those runs to some specs. It never mutates anything."
|
|
105
|
+
"runs passed; --test PATH limits those runs to some specs. It never mutates anything. " \
|
|
106
|
+
"It certifies no versions of Rails, databases or tools saboteur never invokes (#318).",
|
|
67
107
|
takes_options: true,
|
|
68
108
|
exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
|
|
69
109
|
examples: [["Check the setup before the first run", "saboteur doctor"],
|
|
@@ -92,6 +132,53 @@ module Saboteur
|
|
|
92
132
|
["Also fail on a score drop of more than 2 points",
|
|
93
133
|
"saboteur baseline check --baseline baseline.json --epsilon 2"]]
|
|
94
134
|
),
|
|
135
|
+
Command.new(
|
|
136
|
+
name: "survivors",
|
|
137
|
+
summary: "Browse a report's survivors in the terminal, read-only",
|
|
138
|
+
description: "Reads the Stryker JSON report without booting the project or rerunning tests. " \
|
|
139
|
+
"Lists every survivor with rank, file, line, operator, digest and triage label; " \
|
|
140
|
+
"filters with --file, --operator and --triage; --ranked uses the report's ranked " \
|
|
141
|
+
"findings order. --show DIGEST prints the diff, the exact replay command and an " \
|
|
142
|
+
"ignore-file entry template for that mutant. It never edits the ignore file.",
|
|
143
|
+
takes_options: false,
|
|
144
|
+
exit_codes: { 0 => "Survivors listed or shown (including none).",
|
|
145
|
+
2 => "Missing, malformed or unreadable report; bad arguments." },
|
|
146
|
+
examples: [["List survivors from the last run", "saboteur survivors .saboteur/report.json"],
|
|
147
|
+
["Review-priority order", "saboteur survivors .saboteur/report.json --ranked"],
|
|
148
|
+
["Diff, replay command and ignore template for one mutant",
|
|
149
|
+
"saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
|
|
150
|
+
),
|
|
151
|
+
Command.new(
|
|
152
|
+
name: "badge",
|
|
153
|
+
summary: "Print a shields.io endpoint badge over the opt-in score history",
|
|
154
|
+
description: "Reads the score history JSONL (c.score_history, or --history PATH) and prints one " \
|
|
155
|
+
"shields.io endpoint badge JSON for the latest record: label, mutation score and " \
|
|
156
|
+
"color (brightgreen at or above --high, default 80; yellow at or above --low, " \
|
|
157
|
+
"default 60; red below). --output PATH writes the file instead of printing. An " \
|
|
158
|
+
"incomplete latest record is refused unless --mark-partial labels the badge as " \
|
|
159
|
+
"partial. No hosted service, no project boot, no tests run.",
|
|
160
|
+
takes_options: true,
|
|
161
|
+
exit_codes: { 0 => "Badge JSON written.",
|
|
162
|
+
2 => "No history path or unexpected arguments.",
|
|
163
|
+
3 => "Empty or unreadable history, an incomplete record without --mark-partial, " \
|
|
164
|
+
"or no tested mutants." },
|
|
165
|
+
examples: [["Badge the recorded score", "saboteur badge"],
|
|
166
|
+
["Write the badge file CI uploads", "saboteur badge --output .saboteur/badge.json"]]
|
|
167
|
+
),
|
|
168
|
+
Command.new(
|
|
169
|
+
name: "trend",
|
|
170
|
+
summary: "Print the score trend over the opt-in score history",
|
|
171
|
+
description: "Reads the score history JSONL (c.score_history, or --history PATH) and prints the " \
|
|
172
|
+
"latest record next to the most recent comparable one (same configured selection, " \
|
|
173
|
+
"operator policy and a complete run), with the covered-score change in percentage " \
|
|
174
|
+
"points. With no comparable earlier record it says so instead of inventing a " \
|
|
175
|
+
"delta. No hosted service, no project boot, no tests run.",
|
|
176
|
+
takes_options: true,
|
|
177
|
+
exit_codes: { 0 => "Trend printed.",
|
|
178
|
+
2 => "No history path or unexpected arguments.",
|
|
179
|
+
3 => "Empty or unreadable history." },
|
|
180
|
+
examples: [["Show the latest score and its change", "saboteur trend"]]
|
|
181
|
+
),
|
|
95
182
|
Command.new(
|
|
96
183
|
name: "version",
|
|
97
184
|
summary: "Print the installed version",
|
|
@@ -122,14 +209,14 @@ module Saboteur
|
|
|
122
209
|
|
|
123
210
|
AGENT_EXIT_CODES = " With --agent the codes are stricter: 0 only when the run was complete and clean; " \
|
|
124
211
|
"1 survivors to review;\n 2 usage or config error; 3 incomplete (no coverage, caps, errors, " \
|
|
125
|
-
"nothing selected) or tool failure."
|
|
212
|
+
"a budget stop before any mutant ran, nothing selected) or tool failure."
|
|
126
213
|
|
|
127
214
|
FILES = [
|
|
128
215
|
["config/saboteur.rb", "Settings as Ruby (Saboteur.configure). Command line options override it."],
|
|
129
216
|
["config/saboteur_ignore.rb", "Accepted survivors: mutation digests with subject and a required reason."],
|
|
130
217
|
[".saboteur/report.json", "Stryker mutation-testing-report-schema JSON (--json changes the path)."],
|
|
131
218
|
[".saboteur/report.html", "Self-contained HTML report (--html changes the path)."],
|
|
132
|
-
[".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete."]
|
|
219
|
+
[".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete between runs; stop the run first."]
|
|
133
220
|
].freeze
|
|
134
221
|
|
|
135
222
|
# option_lines: the option lines of each command's parser, keyed by command name.
|
|
@@ -166,8 +253,16 @@ module Saboteur
|
|
|
166
253
|
@commands.select(&:takes_options).map { |command| options_section(command) }
|
|
167
254
|
end
|
|
168
255
|
|
|
256
|
+
# run, audit and pr share one exit-code table; the overview keeps every
|
|
257
|
+
# command's section but carries the agent note only on the first table
|
|
258
|
+
# that shares it, so the phrase appears exactly once.
|
|
169
259
|
def exit_code_sections
|
|
170
|
-
|
|
260
|
+
noted = []
|
|
261
|
+
@commands.select(&:exit_codes).map do |command|
|
|
262
|
+
first = noted.none? { |codes| codes.equal?(command.exit_codes) }
|
|
263
|
+
noted << command.exit_codes
|
|
264
|
+
exit_codes_section(command.exit_codes, command.name, agent_note: first)
|
|
265
|
+
end
|
|
171
266
|
end
|
|
172
267
|
|
|
173
268
|
def title
|
|
@@ -204,10 +299,10 @@ module Saboteur
|
|
|
204
299
|
"EXAMPLES\n#{examples.map { |note, line| " # #{note}\n #{line}" }.join("\n")}\n"
|
|
205
300
|
end
|
|
206
301
|
|
|
207
|
-
def exit_codes_section(codes, command_name = nil)
|
|
302
|
+
def exit_codes_section(codes, command_name = nil, agent_note: true)
|
|
208
303
|
rows = codes.map { |code, meaning| " #{code} #{meaning}" }
|
|
209
304
|
title = command_name ? "EXIT CODES (saboteur #{command_name})" : "EXIT CODES"
|
|
210
|
-
note = codes.equal?(EXIT_CODE_MEANINGS) ? "#{AGENT_EXIT_CODES}\n" : ""
|
|
305
|
+
note = agent_note && codes.equal?(EXIT_CODE_MEANINGS) ? "#{AGENT_EXIT_CODES}\n" : ""
|
|
211
306
|
"#{title}\n#{rows.join("\n")}\n#{note}"
|
|
212
307
|
end
|
|
213
308
|
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -21,7 +21,15 @@ module Saboteur
|
|
|
21
21
|
#
|
|
22
22
|
# reason is mandatory; a file with a missing or blank reason does not load.
|
|
23
23
|
class IgnoreList
|
|
24
|
-
|
|
24
|
+
# Triage states beyond plain ignoring (#155): accepted_equivalent and
|
|
25
|
+
# wont_fix suppress the survivor's exit failure (it still counts in the
|
|
26
|
+
# score), todo records the decision to fix and lets the survivor count.
|
|
27
|
+
STATES = %w[accepted_equivalent wont_fix todo].freeze
|
|
28
|
+
|
|
29
|
+
Entry = Data.define(:subject, :mutation, :reason, :state) do
|
|
30
|
+
# Entries built without a state predate triage states: they accept.
|
|
31
|
+
def initialize(subject:, mutation:, reason:, state: "accepted_equivalent") = super
|
|
32
|
+
end
|
|
25
33
|
|
|
26
34
|
# Collects the entries a saboteur_ignore.rb file declares.
|
|
27
35
|
class Collector
|
|
@@ -31,16 +39,23 @@ module Saboteur
|
|
|
31
39
|
@entries = []
|
|
32
40
|
end
|
|
33
41
|
|
|
34
|
-
def mutation(digest = nil, subject: nil, reason: nil)
|
|
42
|
+
def mutation(digest = nil, subject: nil, reason: nil, state: nil)
|
|
35
43
|
line = caller_locations(1, 1).first.lineno
|
|
36
44
|
values = { "subject" => subject, "mutation" => digest, "reason" => reason }
|
|
37
45
|
missing = values.select { |_, value| value.to_s.strip.empty? }.keys
|
|
38
46
|
raise ConfigError, "#{line}: entry is missing #{missing.join(', ')}" if missing.any?
|
|
39
47
|
|
|
48
|
+
state = state.to_s.strip
|
|
49
|
+
state = "accepted_equivalent" if state.empty?
|
|
50
|
+
unless STATES.include?(state)
|
|
51
|
+
raise ConfigError, "#{line}: unknown state #{state.inspect} - use one of #{STATES.join(', ')}"
|
|
52
|
+
end
|
|
53
|
+
|
|
40
54
|
@entries << Entry.new(
|
|
41
55
|
subject: subject.to_s.strip,
|
|
42
56
|
mutation: digest.to_s.strip,
|
|
43
|
-
reason: reason.to_s.strip
|
|
57
|
+
reason: reason.to_s.strip,
|
|
58
|
+
state: state
|
|
44
59
|
)
|
|
45
60
|
end
|
|
46
61
|
end
|
|
@@ -96,14 +111,21 @@ module Saboteur
|
|
|
96
111
|
@index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
|
|
97
112
|
end
|
|
98
113
|
|
|
99
|
-
# True when this result is a survivor the list accepts
|
|
114
|
+
# True when this result is a survivor the list accepts: a todo entry
|
|
115
|
+
# records the triage but keeps the survivor counting as unignored.
|
|
100
116
|
def accepts?(result)
|
|
101
|
-
|
|
117
|
+
return false unless result.survived?
|
|
118
|
+
|
|
119
|
+
entry = @index[[result.subject.id, result.digest]]
|
|
120
|
+
!entry.nil? && entry.state != "todo"
|
|
102
121
|
end
|
|
103
122
|
|
|
104
123
|
# Marks matching survivors as ignored. Returns warnings for stale entries:
|
|
105
124
|
# entries whose subject was examined but whose mutation no longer exists.
|
|
106
|
-
|
|
125
|
+
# unscoped_digests (subject id => every digest generated before a diff
|
|
126
|
+
# scope cut) keeps a scoped run from calling a scoped-out entry stale
|
|
127
|
+
# (#284).
|
|
128
|
+
def apply(results, unscoped_digests: nil)
|
|
107
129
|
seen = {}
|
|
108
130
|
results.each do |result|
|
|
109
131
|
key = [result.subject.id, result.digest]
|
|
@@ -111,10 +133,21 @@ module Saboteur
|
|
|
111
133
|
next unless entry
|
|
112
134
|
|
|
113
135
|
seen[key] = true
|
|
114
|
-
|
|
136
|
+
next unless result.survived?
|
|
137
|
+
|
|
138
|
+
result.triage_state = entry.state
|
|
139
|
+
result.triage_reason = entry.reason
|
|
140
|
+
result.ignored = entry.reason if entry.state != "todo"
|
|
115
141
|
end
|
|
142
|
+
stale_warnings(results, seen, unscoped_digests)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
def stale_warnings(results, seen, unscoped_digests)
|
|
116
148
|
examined = results.to_set { |result| result.subject.id }
|
|
117
149
|
entries.select { |entry| examined.include?(entry.subject) && !seen[[entry.subject, entry.mutation]] }
|
|
150
|
+
.reject { |entry| unscoped_digests&.fetch(entry.subject, nil)&.include?(entry.mutation) }
|
|
118
151
|
.map { |entry| "stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation" }
|
|
119
152
|
end
|
|
120
153
|
end
|
data/lib/saboteur/install.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "fileutils"
|
|
5
|
+
require_relative "text"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
# Writes the Rails starter files: config/saboteur.rb, bin/saboteur and a
|
|
@@ -64,7 +65,7 @@ module Saboteur
|
|
|
64
65
|
# Printed after install; the Gemfile is never edited.
|
|
65
66
|
def gemfile_hint
|
|
66
67
|
gemfile = File.join(@root, "Gemfile")
|
|
67
|
-
return nil if File.file?(gemfile) && File.read(gemfile).match?(/^\s*gem\s+["']saboteur["']/)
|
|
68
|
+
return nil if File.file?(gemfile) && Text.utf8(File.read(gemfile)).match?(/^\s*gem\s+["']saboteur["']/)
|
|
68
69
|
|
|
69
70
|
GEMFILE_HINT
|
|
70
71
|
end
|
|
@@ -108,8 +109,8 @@ module Saboteur
|
|
|
108
109
|
link = symlink_component(".gitignore")
|
|
109
110
|
return Result.new(:skipped, ".gitignore") if link && !@force
|
|
110
111
|
|
|
111
|
-
current = File.file?(path) ? File.read(path) : ""
|
|
112
|
-
listed = listed?(current)
|
|
112
|
+
current = File.file?(path) ? Text.utf8_identity(File.read(path)) : ""
|
|
113
|
+
listed = listed?(Text.utf8(current))
|
|
113
114
|
return Result.new(:unchanged, ".gitignore") if listed && !link
|
|
114
115
|
|
|
115
116
|
File.delete(link) if link
|
|
@@ -10,8 +10,13 @@ module Saboteur
|
|
|
10
10
|
# so a kill by a failed assertion can be told from a kill by a crash.
|
|
11
11
|
# failure_messages: the first line of each failure, so a verdict that
|
|
12
12
|
# does not hold up can say why the test failed.
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
# failure_details: the full message and first backtrace frames of each
|
|
14
|
+
# failure, so a neutral re-run (the verdict check) can say exactly why
|
|
15
|
+
# the killer failed with the method unchanged (#296).
|
|
16
|
+
RunResult = Data.define(:passed, :failed_ids, :executed_ids, :runtime, :failure_classes, :failure_messages,
|
|
17
|
+
:failure_details) do
|
|
18
|
+
def initialize(passed:, failed_ids:, executed_ids:, runtime:, failure_classes: {}, failure_messages: {},
|
|
19
|
+
failure_details: {})
|
|
15
20
|
super
|
|
16
21
|
end
|
|
17
22
|
end
|