saboteur 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +215 -0
- data/README.md +44 -6
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +116 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +6 -1
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +29 -305
- data/lib/saboteur/config.rb +76 -7
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +24 -5
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +6 -4
- data/lib/saboteur/doctor.rb +10 -0
- data/lib/saboteur/engine/baseline_guard.rb +118 -15
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +16 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +296 -21
- data/lib/saboteur/engine/diagnostics.rb +24 -7
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +4 -1
- data/lib/saboteur/engine/load_coverage.rb +72 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +11 -3
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +10 -7
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +26 -5
- data/lib/saboteur/engine/scheduler.rb +33 -25
- data/lib/saboteur/engine/selection_announcement.rb +9 -3
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +144 -64
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +186 -32
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +44 -11
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +123 -1
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +39 -20
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +42 -8
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +145 -18
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +33 -21
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +13 -0
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +45 -5
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +150 -9
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +9 -0
- metadata +48 -16
|
@@ -64,7 +64,7 @@ module Saboteur
|
|
|
64
64
|
results = run_candidates(candidates) do |mutation, text|
|
|
65
65
|
reuse(plan, mutation, text) || execute(plan, mutation, text)
|
|
66
66
|
end
|
|
67
|
-
checked(plan, results)
|
|
67
|
+
checked(plan, results, truncated: results.size < candidates.size)
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
# The per-mutant timeout: timeout_factor times what the tests took
|
|
@@ -81,14 +81,21 @@ module Saboteur
|
|
|
81
81
|
|
|
82
82
|
# Verdicts that did not hold up are warned about once per subject;
|
|
83
83
|
# only checked verdicts are stored for --incremental.
|
|
84
|
-
# A spent budget admits no check
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
# reports
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
84
|
+
# A spent budget admits no check for a subject that finished its
|
|
85
|
+
# mutants: forking the unchanged run with no time left would turn
|
|
86
|
+
# genuine kills into fabricated errors and cache them. Fresh kills
|
|
87
|
+
# stand unverified instead - outside the score, never cached - and
|
|
88
|
+
# the distribution marks the subject so the run reports
|
|
89
|
+
# budget-partial.
|
|
90
|
+
# A subject the budget stopped mid-list is different (#588): its
|
|
91
|
+
# completed mutants were already measured, so their checks still
|
|
92
|
+
# run, uncapped - the budget stops new mutation work, not the
|
|
93
|
+
# confirmation of work already done. The overshoot stays bounded by
|
|
94
|
+
# the subject's distinct killing sequences.
|
|
95
|
+
def checked(plan, results, truncated: false)
|
|
96
|
+
return results.map { |result| unverifiable(result) } if budget_spent? && !truncated
|
|
97
|
+
|
|
98
|
+
@confirming_truncation = truncated
|
|
92
99
|
subject = plan.subject
|
|
93
100
|
check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
|
|
94
101
|
baseline_timeout: @config["baseline_timeout"])
|
|
@@ -101,6 +108,8 @@ module Saboteur
|
|
|
101
108
|
@diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
|
|
102
109
|
end
|
|
103
110
|
final
|
|
111
|
+
ensure
|
|
112
|
+
@confirming_truncation = false
|
|
104
113
|
end
|
|
105
114
|
|
|
106
115
|
# A kill rejected because its covering examples fail with the method
|
|
@@ -111,10 +120,12 @@ module Saboteur
|
|
|
111
120
|
# mutant's run on the same examples. Otherwise the error stays.
|
|
112
121
|
#
|
|
113
122
|
# Cost bound: at most two extra forks per rejected kill, the unchanged
|
|
114
|
-
# prefix run and one mutant run.
|
|
115
|
-
# other fork: neither starts once the budget is spent, and
|
|
116
|
-
# more than the time left. A retry the budget stops is
|
|
117
|
-
# an error or a kill.
|
|
123
|
+
# prefix run and one mutant run. Ordinarily both spend --max-duration
|
|
124
|
+
# like every other fork: neither starts once the budget is spent, and
|
|
125
|
+
# neither gets more than the time left. A retry the budget stops is
|
|
126
|
+
# unverified, not an error or a kill. The exception is a truncated
|
|
127
|
+
# subject (#588): its completed kills still retry, at the full
|
|
128
|
+
# timeout. Keep this bound if the retry ever grows.
|
|
118
129
|
def retried_in_defined_order(plan, result)
|
|
119
130
|
return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
|
|
120
131
|
|
|
@@ -129,10 +140,10 @@ module Saboteur
|
|
|
129
140
|
end
|
|
130
141
|
|
|
131
142
|
def rerun_in_prefix(plan, result, tests)
|
|
132
|
-
return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent?
|
|
143
|
+
return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent? && !@confirming_truncation
|
|
133
144
|
|
|
134
145
|
full = timeout_for(tests, plan.slowdown)
|
|
135
|
-
capped = budget_capped(full)
|
|
146
|
+
capped = @confirming_truncation ? full : budget_capped(full)
|
|
136
147
|
outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
|
|
137
148
|
timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
|
|
138
149
|
return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
|
|
@@ -150,11 +161,11 @@ module Saboteur
|
|
|
150
161
|
# the check never got its answer (#296).
|
|
151
162
|
def unchanged_runner(subject, plan)
|
|
152
163
|
lambda do |tests, timeout|
|
|
153
|
-
if budget_spent?
|
|
164
|
+
if budget_spent? && !@confirming_truncation
|
|
154
165
|
Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
|
|
155
166
|
message: "the --max-duration budget is spent")
|
|
156
167
|
else
|
|
157
|
-
capped = budget_capped(timeout)
|
|
168
|
+
capped = @confirming_truncation ? timeout : budget_capped(timeout)
|
|
158
169
|
outcome = @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: capped,
|
|
159
170
|
eval_prefix: eval_prefix(plan))
|
|
160
171
|
note_leftovers(subject, "verdict check", outcome)
|
|
@@ -238,17 +249,14 @@ module Saboteur
|
|
|
238
249
|
[@config["timeout_factor"] * runtime, @config["timeout_min"]].max
|
|
239
250
|
end
|
|
240
251
|
|
|
241
|
-
# What a subject no test ran gets.
|
|
242
|
-
#
|
|
243
|
-
# it, so it gets its own status, out of the score, with a hint.
|
|
252
|
+
# What a subject no test ran gets. Load-only code is not "no coverage":
|
|
253
|
+
# it ran, but no forked test can re-run it, so it gets its own status.
|
|
244
254
|
def unloadable(candidates, subject, line_numbers = nil)
|
|
255
|
+
return skip_all(candidates, subject, Selection::CoverageMap::PRELOADED) if @map.preloaded?(subject.file)
|
|
245
256
|
return skip_all(candidates, subject, "no coverage") unless
|
|
246
257
|
@map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
|
|
247
258
|
|
|
248
|
-
candidates
|
|
249
|
-
MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
|
|
250
|
-
reason: Selection::CoverageMap::RAN_AT_LOAD)
|
|
251
|
-
end
|
|
259
|
+
load_time_results(candidates, subject)
|
|
252
260
|
end
|
|
253
261
|
|
|
254
262
|
# A def-time macro's only callers ran while the app loaded: the
|
|
@@ -329,7 +337,7 @@ module Saboteur
|
|
|
329
337
|
# Each mutant is checked separately: changing a body can change which
|
|
330
338
|
# definition a test reaches, so one unreachable verdict cannot skip later work.
|
|
331
339
|
def note_unreachable(plan, outcome)
|
|
332
|
-
@diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
|
|
340
|
+
@diagnostics.warn("#{plan.subject.id}: a mutant never ran and is not scored\n #{outcome.message}")
|
|
333
341
|
end
|
|
334
342
|
|
|
335
343
|
# The text a known def-time wrapper needs re-evaluated with the def.
|
|
@@ -10,10 +10,15 @@ module Saboteur
|
|
|
10
10
|
module SelectionAnnouncement
|
|
11
11
|
private
|
|
12
12
|
|
|
13
|
+
# A --since diff with no changed source file (a spec-only or docs-only
|
|
14
|
+
# change) yields zero subjects, so the suite never runs: say there is
|
|
15
|
+
# nothing to test rather than promise a whole-suite run that does not
|
|
16
|
+
# happen (#607).
|
|
13
17
|
def selection_message(guess)
|
|
14
|
-
if guess.changed.zero?
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
if guess.changed.zero? && config["since"]
|
|
19
|
+
"inferred specs: no changed source file to infer a spec for, so there is nothing to test"
|
|
20
|
+
elsif guess.changed.zero?
|
|
21
|
+
"inferred specs: no source file to infer a spec for, so the whole suite runs"
|
|
17
22
|
elsif guess.specs.empty?
|
|
18
23
|
guess.no_specs_message(since: config["since"])
|
|
19
24
|
else
|
|
@@ -47,6 +52,7 @@ module Saboteur
|
|
|
47
52
|
# opt-out and would warn.
|
|
48
53
|
@config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
|
|
49
54
|
@guessed_tests = guess.specs
|
|
55
|
+
@guessed_referenced = guess.referenced
|
|
50
56
|
guess.scope_message(since: config["since"])
|
|
51
57
|
end
|
|
52
58
|
end
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Compares the map a partial rebuild produced with the full map (#46,
|
|
7
|
+
# step 3). The full map is the one a run uses; this only says whether the
|
|
8
|
+
# partial path would have been right.
|
|
9
|
+
#
|
|
10
|
+
# Two things are compared, and both must match:
|
|
11
|
+
# - the semantic evidence: which tests exist and, per source file, which
|
|
12
|
+
# tests hit which lines, methods and branch arms, and which lines ran
|
|
13
|
+
# while the app loaded. Durations, build time and the order of ids are
|
|
14
|
+
# left out, as are entries with no test.
|
|
15
|
+
# - what the result cache would watch: for every test the source files
|
|
16
|
+
# it touched, plus the boot files and the loaded files nothing
|
|
17
|
+
# observed. Evidence that matches line by line but yields another
|
|
18
|
+
# dependency set is worse than a line mismatch, since the set decides
|
|
19
|
+
# whether an old verdict is reused.
|
|
20
|
+
module ShadowComparison
|
|
21
|
+
MAX_DIFFERENCES = 50
|
|
22
|
+
KINDS = %i[lines methods branches].freeze
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
# { result: :match | :superset | :mismatch, match: bool, total: Integer, extra: Integer,
|
|
27
|
+
# gained: Integer,
|
|
28
|
+
# differences: [{kind:, name:}] } with at most MAX_DIFFERENCES entries,
|
|
29
|
+
# sorted by kind and name.
|
|
30
|
+
#
|
|
31
|
+
# :superset means the partial map credits tests the full map does not,
|
|
32
|
+
# on lines the full map also credits, and never lacks a credit the
|
|
33
|
+
# full map has. Anything else that differs is :mismatch, including an
|
|
34
|
+
# extra credit on a line the full map calls uncovered: it would turn
|
|
35
|
+
# "no test covers this" into "covered". Those are counted apart, in
|
|
36
|
+
# gained. extra counts the other extra credits (one per file, line and
|
|
37
|
+
# test) in lines, methods and branches.
|
|
38
|
+
def call(partial, full, fallback_files: [])
|
|
39
|
+
maps = [partial, full]
|
|
40
|
+
found = evidence_differences(maps) + dependency_differences(maps, fallback_files)
|
|
41
|
+
counted = found.select { |item| KINDS.map(&:to_s).include?(item[:kind]) }
|
|
42
|
+
extra = counted.sum { |item| item[:extra] }
|
|
43
|
+
gained = counted.sum { |item| item[:gained] }
|
|
44
|
+
result = if found.empty? then :match
|
|
45
|
+
elsif gained.positive? || found.any? { |item| item[:missing] } then :mismatch
|
|
46
|
+
else :superset
|
|
47
|
+
end
|
|
48
|
+
shown = found.sort_by { |item| [item[:kind], item[:name]] }.first(MAX_DIFFERENCES)
|
|
49
|
+
{ result: result, match: result == :match, total: found.size, extra: extra, gained: gained,
|
|
50
|
+
differences: shown.map { |item| item.slice(:kind, :name) } }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def evidence_differences(maps)
|
|
54
|
+
partial, full = maps
|
|
55
|
+
found = []
|
|
56
|
+
unless partial.tests.sort == full.tests.sort
|
|
57
|
+
found << { kind: "tests", name: "the set of test ids", missing: true }
|
|
58
|
+
end
|
|
59
|
+
KINDS.each { |kind| found.concat(hit_differences(kind.to_s, hits(partial, kind), hits(full, kind))) }
|
|
60
|
+
found.concat(exact_differences("load", load_hits(partial), load_hits(full)))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Per name: the credits only the partial map has (extra) and the ones
|
|
64
|
+
# only the full map has (missing).
|
|
65
|
+
def hit_differences(kind, left, right)
|
|
66
|
+
(left.keys | right.keys).filter_map do |name|
|
|
67
|
+
extra, gained, missing = credits(left[name] || {}, right[name] || {})
|
|
68
|
+
next if extra.zero? && gained.zero? && !missing
|
|
69
|
+
|
|
70
|
+
{ kind: kind, name: name, extra: extra, gained: gained, missing: missing }
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def credits(left, right)
|
|
75
|
+
extra = gained = 0
|
|
76
|
+
missing = false
|
|
77
|
+
(left.keys | right.keys).each do |line|
|
|
78
|
+
mine = left[line] || []
|
|
79
|
+
theirs = right[line] || []
|
|
80
|
+
(theirs.empty? ? gained += mine.size : extra += (mine - theirs).size)
|
|
81
|
+
missing ||= (theirs - mine).any?
|
|
82
|
+
end
|
|
83
|
+
[extra, gained, missing]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def exact_differences(kind, left, right)
|
|
87
|
+
(left.keys | right.keys).reject { |name| left[name] == right[name] }
|
|
88
|
+
.map { |name| { kind: kind, name: name, missing: true } }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# {file => {line-or-span => sorted ids}} without empty entries.
|
|
92
|
+
def hits(map, kind)
|
|
93
|
+
map.public_send(kind).each_with_object({}) do |(file, by_line), acc|
|
|
94
|
+
kept = by_line.transform_values(&:sort).reject { |_line, ids| ids.empty? }
|
|
95
|
+
acc[file] = kept unless kept.empty?
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def load_hits(map)
|
|
100
|
+
[map.load_lines, map.load_methods].flat_map(&:to_a).each_with_object({}) do |(file, lines), acc|
|
|
101
|
+
acc[file] = (acc.fetch(file, []) | lines).sort unless lines.empty?
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The files every verdict depends on must match exactly. What each
|
|
106
|
+
# test depends on follows from its line and method credits, which are
|
|
107
|
+
# compared above, so it is not compared again.
|
|
108
|
+
def dependency_differences(maps, fallback_files)
|
|
109
|
+
partial, full = maps
|
|
110
|
+
left = dependencies(partial, fallback_files)
|
|
111
|
+
right = dependencies(full, fallback_files)
|
|
112
|
+
left.keys.reject { |name| left[name] == right[name] }.map { |name| { kind: "deps", name: name, missing: true } }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def dependencies(map, fallback_files)
|
|
116
|
+
unobserved = (fallback_files | map.loaded_files).select { |file| map.empty_for?([file]) }.sort
|
|
117
|
+
{ "(boot files)" => map.boot_files, "(unobserved files)" => unobserved }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# One shadow comparison (#46, step 3): would a partial rebuild from the
|
|
7
|
+
# last map have matched the full map this run just built? The run uses
|
|
8
|
+
# the full map either way; this only produces the report.
|
|
9
|
+
#
|
|
10
|
+
# The report always carries the share threshold and, when the partial
|
|
11
|
+
# path was possible, the observed stale share, so the constant can be
|
|
12
|
+
# fitted from data later.
|
|
13
|
+
module ShadowRun
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# base: the earlier inputs snapshot and its map. current: the inputs
|
|
17
|
+
# snapshot of this run. rerun.call(stale_spec_files, seed) builds a map
|
|
18
|
+
# over just those files. clean turns a name into a line-safe string.
|
|
19
|
+
def call(base:, base_map:, current:, full_map:, invalidation:, fallback_files:, rerun:, clean:)
|
|
20
|
+
report = { "threshold" => PartialInvalidation::THRESHOLD, "base_map" => base["map_key"] }
|
|
21
|
+
stale = invalidation.stale_specs(base, current)
|
|
22
|
+
return report.merge("result" => "skipped", "reason" => "a full rebuild is needed") if stale == :full
|
|
23
|
+
|
|
24
|
+
merged = stale.empty? ? base_map : PartialMerge.call(base_map, rerun.call(stale, base["seed"]), stale)
|
|
25
|
+
verdict = ShadowComparison.call(merged, full_map, fallback_files: fallback_files)
|
|
26
|
+
report.merge("result" => verdict[:result].to_s, "extra_credits" => verdict[:extra],
|
|
27
|
+
"gained_credits" => verdict[:gained],
|
|
28
|
+
"stale" => stale.map(&clean),
|
|
29
|
+
"stale_share" => invalidation.share(stale), "total" => verdict[:total],
|
|
30
|
+
"differences" => verdict[:differences].map do |item|
|
|
31
|
+
item.to_h do |key, value|
|
|
32
|
+
[key.to_s, clean.call(value)]
|
|
33
|
+
end
|
|
34
|
+
end)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -4,12 +4,39 @@
|
|
|
4
4
|
require_relative "../text"
|
|
5
5
|
require_relative "../constant_subject"
|
|
6
6
|
require_relative "../constant_subject_finder"
|
|
7
|
+
require_relative "../attribute_finder"
|
|
7
8
|
require_relative "../constant_reads"
|
|
8
9
|
|
|
9
10
|
module Saboteur
|
|
10
11
|
class Engine
|
|
11
12
|
# The subjects to mutate: every def in the included files, narrowed by
|
|
12
13
|
# --subject patterns and by --since (methods with a changed line).
|
|
14
|
+
# A heredoc body is code the static subject walk cannot see: methods
|
|
15
|
+
# defined by class_eval'ing the string, or behavior living in a
|
|
16
|
+
# template constant, exist only when the string is evaluated (#467).
|
|
17
|
+
module HeredocSpans
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
def changed_lines_inside?(file, changed_lines)
|
|
21
|
+
result = Prism.parse(File.binread(file))
|
|
22
|
+
return false if result.failure?
|
|
23
|
+
|
|
24
|
+
spans = collect(result.value)
|
|
25
|
+
changed_lines.any? { |line| spans.any? { |span| span.cover?(line) } }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def collect(node, spans = [])
|
|
29
|
+
if (node.is_a?(Prism::StringNode) || node.is_a?(Prism::InterpolatedStringNode)) &&
|
|
30
|
+
node.opening_loc&.slice&.start_with?("<<")
|
|
31
|
+
# The node location of an interpolated string stops at the opening
|
|
32
|
+
# marker; opening through closing covers the whole heredoc body.
|
|
33
|
+
spans << (node.opening_loc.start_line..node.closing_loc.end_line)
|
|
34
|
+
end
|
|
35
|
+
node.compact_child_nodes.each { |child| collect(child, spans) }
|
|
36
|
+
spans
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
13
40
|
class SubjectSelection
|
|
14
41
|
# Lines changed since the --since ref, by file; nil without --since.
|
|
15
42
|
attr_reader :changed_lines
|
|
@@ -33,12 +60,36 @@ module Saboteur
|
|
|
33
60
|
subjects += constant_subjects
|
|
34
61
|
end
|
|
35
62
|
@discovered = subjects
|
|
63
|
+
warn_unmatched_whole_file(subjects) if @config["since"]
|
|
36
64
|
subjects = by_pattern(subjects, @config["subjects"]) if @config["subjects"].any?
|
|
37
65
|
subjects = by_change(subjects, @config["since"]) if @config["since"]
|
|
38
66
|
subjects = drop_intentionally_untested(subjects)
|
|
67
|
+
inform_nothing_to_test(subjects)
|
|
39
68
|
attach_consumers(subjects)
|
|
40
69
|
end
|
|
41
70
|
|
|
71
|
+
# A --since run can legitimately select nothing: the range is empty
|
|
72
|
+
# (a checkout AT the base, #643) or its changes carry no mutable Ruby
|
|
73
|
+
# (a docs-only diff, #659). Exit 0 with a notice, not a silent green:
|
|
74
|
+
# name the base, the changed-file count and why nothing was selected.
|
|
75
|
+
def inform_nothing_to_test(subjects)
|
|
76
|
+
ref = @config["since"]
|
|
77
|
+
return unless ref && subjects.empty?
|
|
78
|
+
|
|
79
|
+
changed = Selection::GitDiff.changed_paths(ref, root: @root)
|
|
80
|
+
count = changed[:present].size + changed[:deleted].size
|
|
81
|
+
reason = if count.zero?
|
|
82
|
+
"the resolved range has no changes"
|
|
83
|
+
elsif !@raw_changed_files.intersect?(@files)
|
|
84
|
+
"#{count} changed file#{'s' unless count == 1}, none in mutation scope " \
|
|
85
|
+
"(docs-only, excluded, or non-Ruby)"
|
|
86
|
+
else
|
|
87
|
+
"no subject covers the changed lines (layout-only edits, deletions, " \
|
|
88
|
+
"or intentionally untested code)"
|
|
89
|
+
end
|
|
90
|
+
@diagnostics.inform("--since #{ref}: nothing to test - #{reason}")
|
|
91
|
+
end
|
|
92
|
+
|
|
42
93
|
# "Klass#meth", "Klass.meth", or a namespace prefix ("Klass", "A::B").
|
|
43
94
|
def self.match?(id, pattern)
|
|
44
95
|
id == pattern || %w[# . ::].any? { |separator| id.start_with?("#{pattern}#{separator}") }
|
|
@@ -83,7 +134,8 @@ module Saboteur
|
|
|
83
134
|
else
|
|
84
135
|
@diagnostics.record_discovery_failure(file, reason: "invalid_file_syntax")
|
|
85
136
|
parsed.errors.first(1).each do |message|
|
|
86
|
-
@diagnostics.warn("#{file.delete_prefix("#{@root}/")}
|
|
137
|
+
@diagnostics.warn("fix the syntax in #{Text.display_safe(file.delete_prefix("#{@root}/"))}: " \
|
|
138
|
+
"does not parse (#{Text.display_safe(message)})")
|
|
87
139
|
end
|
|
88
140
|
end
|
|
89
141
|
end
|
|
@@ -96,16 +148,12 @@ module Saboteur
|
|
|
96
148
|
subjects.any? { |subject| self.class.match?(subject.id, pattern) }
|
|
97
149
|
end
|
|
98
150
|
missed.each do |pattern|
|
|
99
|
-
@diagnostics.warn(
|
|
100
|
-
"Klass, Klass#meth or Klass.meth, not globs)")
|
|
151
|
+
@diagnostics.warn(missed_pattern_message(pattern))
|
|
101
152
|
end
|
|
102
153
|
# A total miss is a typo: refuse instead of a green zero-subject
|
|
103
154
|
# summary (#379). A partial miss runs, but the matched count is
|
|
104
155
|
# on the record.
|
|
105
|
-
if missed.size == patterns.size
|
|
106
|
-
raise ConfigError, "--subject #{patterns.join(', ')} matches no subject (patterns are names like " \
|
|
107
|
-
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
108
|
-
end
|
|
156
|
+
raise ConfigError, total_miss_message(missed) if missed.size == patterns.size
|
|
109
157
|
|
|
110
158
|
if missed.any?
|
|
111
159
|
@diagnostics.warn("#{patterns.size - missed.size} of #{patterns.size} --subject patterns matched")
|
|
@@ -113,6 +161,37 @@ module Saboteur
|
|
|
113
161
|
subjects.select { |subject| patterns.any? { |pattern| self.class.match?(subject.id, pattern) } }
|
|
114
162
|
end
|
|
115
163
|
|
|
164
|
+
# The refusal when every pattern missed (#379). A pattern naming an
|
|
165
|
+
# attribute-macro method gets the real reason - the method exists but
|
|
166
|
+
# has no body to mutate (#373) - instead of the pattern-format hint.
|
|
167
|
+
def total_miss_message(missed)
|
|
168
|
+
if missed.all? { |pattern| attribute_macro_for(pattern) }
|
|
169
|
+
return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
"--subject #{missed.join(', ')} matches no subject (patterns are names like " \
|
|
173
|
+
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Why one missed pattern matched nothing: when it names a method an
|
|
177
|
+
# attribute macro generated, say so (#373); otherwise the pattern
|
|
178
|
+
# itself is suspect.
|
|
179
|
+
def missed_pattern_message(pattern)
|
|
180
|
+
macro = attribute_macro_for(pattern)
|
|
181
|
+
return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
|
|
182
|
+
|
|
183
|
+
"fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
|
|
184
|
+
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# The attribute macro that generated the method a pattern names, if
|
|
188
|
+
# any. The files are parsed for macros once, only when a pattern
|
|
189
|
+
# missed.
|
|
190
|
+
def attribute_macro_for(pattern)
|
|
191
|
+
attributes = @attributes ||= @files.flat_map { |file| AttributeFinder.new.call(file) }
|
|
192
|
+
attributes.find { |found| found.id == pattern }&.macro
|
|
193
|
+
end
|
|
194
|
+
|
|
116
195
|
def by_change(subjects, ref)
|
|
117
196
|
changed = changed_subjects(subjects)
|
|
118
197
|
warn_changed_files_without_subjects
|
|
@@ -188,12 +267,14 @@ module Saboteur
|
|
|
188
267
|
return subjects if reasons.empty?
|
|
189
268
|
|
|
190
269
|
constant_warned_files << canonical
|
|
191
|
-
path = file.delete_prefix("#{@root}/")
|
|
270
|
+
path = Text.display_safe(file.delete_prefix("#{@root}/"))
|
|
192
271
|
listed = reasons.first(2).join(", ")
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
"
|
|
196
|
-
"
|
|
272
|
+
listed += ", and #{reasons.size - 2} more" if reasons.size > 2
|
|
273
|
+
@diagnostics.warn("#{path}: list the constant in c.whole_file to replay it anyway, or move the " \
|
|
274
|
+
"literal into its own file\n --since: constant subject(s) in this file are not " \
|
|
275
|
+
"mutated: it runs code at load (#{listed}) that every constant mutant would run " \
|
|
276
|
+
"again, and saboteur cannot vet what re-running it does. " \
|
|
277
|
+
"#{whole_file_id_hint(rest)}")
|
|
197
278
|
opted_in
|
|
198
279
|
end
|
|
199
280
|
|
|
@@ -203,10 +284,36 @@ module Saboteur
|
|
|
203
284
|
Array(@config["whole_file"]).any? { |pattern| File.fnmatch?(pattern.to_s, subject.id) }
|
|
204
285
|
end
|
|
205
286
|
|
|
287
|
+
# A bare constant name only ever matches a top-level constant's id -
|
|
288
|
+
# never a nested one - so name the exact ids to copy into c.whole_file.
|
|
289
|
+
# The glob example carries its blast radius: it opts in every matching
|
|
290
|
+
# subject project-wide, so the exact id(s) are the safer pick (#635).
|
|
291
|
+
def whole_file_id_hint(subjects)
|
|
292
|
+
ids = subjects.map(&:id)
|
|
293
|
+
named = ids.first(3).map { |id| "\"#{id}\"" }.join(", ")
|
|
294
|
+
named += ", +#{ids.size - 3} more" if ids.size > 3
|
|
295
|
+
"Exact id(s): #{named} (preferred); a glob like \"*#{ids.first.split('::').last}*\" also " \
|
|
296
|
+
"works but opts in every matching subject project-wide."
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# A whole_file pattern that matches nothing silently does nothing (#635):
|
|
300
|
+
# name the pattern and what subject ids look like, so a bare constant
|
|
301
|
+
# name for a nested constant (which never matches its qualified id)
|
|
302
|
+
# stands out. Scoped to this run: a pattern for a file outside the
|
|
303
|
+
# current include set also matches nothing.
|
|
304
|
+
def warn_unmatched_whole_file(subjects)
|
|
305
|
+
@config["whole_file"].each do |pattern|
|
|
306
|
+
next if subjects.any? { |subject| File.fnmatch?(pattern.to_s, subject.id) }
|
|
307
|
+
|
|
308
|
+
@diagnostics.warn("fix the whole_file entry \"#{Text.display_safe(pattern.to_s)}\": it matches no " \
|
|
309
|
+
"discovered subject in this run (subject ids look like RULES, A::B::C or A::B#m)")
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
|
|
206
313
|
def warn_unsupported_constant(found)
|
|
207
|
-
path = found.file.delete_prefix("#{@root}/")
|
|
208
|
-
@diagnostics.warn("--since: changed constant #{found.id} in #{path} is not mutated
|
|
209
|
-
"
|
|
314
|
+
path = Text.display_safe(found.file.delete_prefix("#{@root}/"))
|
|
315
|
+
@diagnostics.warn("--since: changed constant #{found.id} in #{path} is not mutated\n " \
|
|
316
|
+
"#{found.reason}; only hash and array literal entries are mutated")
|
|
210
317
|
end
|
|
211
318
|
|
|
212
319
|
# The tests that can observe a constant mutant are the ones covering a
|
|
@@ -225,9 +332,9 @@ module Saboteur
|
|
|
225
332
|
subject.consumer_defs = reads.transform_values(&:defs)
|
|
226
333
|
if subject.consumer_lines.empty? && reads.any? { |_file, r| r.load.any? }
|
|
227
334
|
path = subject.file.delete_prefix("#{@root}/")
|
|
228
|
-
@diagnostics.warn("--since: changed constant #{subject.id} in #{path} is not mutated
|
|
229
|
-
"reads capture the value at load time in another file, so no test
|
|
230
|
-
"observe a change to it
|
|
335
|
+
@diagnostics.warn("--since: changed constant #{subject.id} in #{path} is not mutated\n " \
|
|
336
|
+
"its only reads capture the value at load time in another file, so no test " \
|
|
337
|
+
"can observe a change to it")
|
|
231
338
|
next
|
|
232
339
|
end
|
|
233
340
|
subject
|
|
@@ -290,11 +397,10 @@ module Saboteur
|
|
|
290
397
|
unmatched = (@changed_lines.keys.select { |file| file.end_with?(".rb") } & included) - matched
|
|
291
398
|
return if unmatched.empty?
|
|
292
399
|
|
|
293
|
-
names =
|
|
294
|
-
more = unmatched.size > 5 ? ", ..." : ""
|
|
400
|
+
names = Text.name_list(unmatched.map { |file| file.delete_prefix("#{@root}/") })
|
|
295
401
|
note = constant_note(unmatched)
|
|
296
402
|
@diagnostics.warn("--since: #{unmatched.size} changed Ruby file(s) have no subjects covering " \
|
|
297
|
-
"the changed lines, so nothing is mutated from them: #{names}
|
|
403
|
+
"the changed lines, so nothing is mutated from them: #{names}\n " \
|
|
298
404
|
"#{subjectless_hint(unmatched)}#{note}")
|
|
299
405
|
end
|
|
300
406
|
|
|
@@ -305,7 +411,7 @@ module Saboteur
|
|
|
305
411
|
constants = unmatched.count { |file| constant_warned_files.include?(file) }
|
|
306
412
|
return "" if constants.zero?
|
|
307
413
|
|
|
308
|
-
"
|
|
414
|
+
"\n #{constants} of these #{constants == 1 ? 'file has' : 'files have'} changes inside " \
|
|
309
415
|
"a constant literal that is not mutated (the constant warnings above name them)."
|
|
310
416
|
end
|
|
311
417
|
|
|
@@ -318,37 +424,15 @@ module Saboteur
|
|
|
318
424
|
embedded = unmatched.select { |file| changed_lines_inside_heredoc?(file) }
|
|
319
425
|
return "The file may not parse, or every change may sit outside a method." if embedded.empty?
|
|
320
426
|
|
|
321
|
-
names =
|
|
322
|
-
|
|
323
|
-
"In #{names}#{more} the changed lines are code embedded in a string (a heredoc, often the " \
|
|
427
|
+
names = Text.name_list(embedded.map { |file| file.delete_prefix("#{@root}/") })
|
|
428
|
+
"In #{names}, the changed lines are code embedded in a string (a heredoc, often the " \
|
|
324
429
|
"argument to class_eval/module_eval or a template constant); methods defined by evaluating " \
|
|
325
430
|
"strings are not mutation subjects, so this run cannot see that behavior."
|
|
326
431
|
end
|
|
327
432
|
|
|
328
|
-
#
|
|
329
|
-
#
|
|
330
|
-
|
|
331
|
-
# Changed lines inside one deserve the specific warning, not the
|
|
332
|
-
# generic parse/method guess. A file that does not parse keeps the
|
|
333
|
-
# generic hint: its problem is named there already.
|
|
334
|
-
def changed_lines_inside_heredoc?(file)
|
|
335
|
-
result = Prism.parse(File.binread(file))
|
|
336
|
-
return false if result.failure?
|
|
337
|
-
|
|
338
|
-
spans = heredoc_spans(result.value)
|
|
339
|
-
@changed_lines[file].any? { |line| spans.any? { |span| span.cover?(line) } }
|
|
340
|
-
end
|
|
341
|
-
|
|
342
|
-
def heredoc_spans(node, spans = [])
|
|
343
|
-
if (node.is_a?(Prism::StringNode) || node.is_a?(Prism::InterpolatedStringNode)) &&
|
|
344
|
-
node.opening_loc&.slice&.start_with?("<<")
|
|
345
|
-
# The node location of an interpolated string stops at the opening
|
|
346
|
-
# marker; opening through closing covers the whole heredoc body.
|
|
347
|
-
spans << (node.opening_loc.start_line..node.closing_loc.end_line)
|
|
348
|
-
end
|
|
349
|
-
node.compact_child_nodes.each { |child| heredoc_spans(child, spans) }
|
|
350
|
-
spans
|
|
351
|
-
end
|
|
433
|
+
# Changed lines inside a heredoc deserve the specific warning, not the
|
|
434
|
+
# generic parse/method guess (#467); the scan lives in HeredocSpans.
|
|
435
|
+
def changed_lines_inside_heredoc?(file) = HeredocSpans.changed_lines_inside?(file, @changed_lines[file])
|
|
352
436
|
|
|
353
437
|
# A changed Ruby file that an include pattern covers but c.exclude
|
|
354
438
|
# removes yields zero candidates with no other sign - this repo's own
|
|
@@ -359,12 +443,11 @@ module Saboteur
|
|
|
359
443
|
hits = excluded_territory & (@changed_lines.keys + untracked)
|
|
360
444
|
return if hits.empty?
|
|
361
445
|
|
|
362
|
-
names = hits.sort.
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
"c.exclude
|
|
366
|
-
"
|
|
367
|
-
"excludes are then ignored).")
|
|
446
|
+
names = Text.name_list(hits.sort.map { |file| file.delete_prefix("#{@root}/") })
|
|
447
|
+
@diagnostics.warn("narrow or remove the c.exclude entry, or pass the file as a positional source " \
|
|
448
|
+
"operand: #{hits.size} changed Ruby file(s) are covered by c.include but removed " \
|
|
449
|
+
"by c.exclude\n --since: nothing is mutated from them: #{names} (configured " \
|
|
450
|
+
"excludes are then ignored)")
|
|
368
451
|
end
|
|
369
452
|
|
|
370
453
|
# Ruby files an include pattern and an exclude pattern both match,
|
|
@@ -389,10 +472,9 @@ module Saboteur
|
|
|
389
472
|
end
|
|
390
473
|
return subjects if dropped.empty?
|
|
391
474
|
|
|
392
|
-
names =
|
|
393
|
-
more = dropped.size > 5 ? ", ..." : ""
|
|
475
|
+
names = Text.name_list(dropped)
|
|
394
476
|
@diagnostics.inform("#{dropped.size} subject(s) skipped: intentionally untested in " \
|
|
395
|
-
"config/saboteur_custom_map.rb: #{names}
|
|
477
|
+
"config/saboteur_custom_map.rb: #{names}")
|
|
396
478
|
kept
|
|
397
479
|
end
|
|
398
480
|
|
|
@@ -417,19 +499,17 @@ module Saboteur
|
|
|
417
499
|
def report_deleted(deleted)
|
|
418
500
|
return if deleted.empty?
|
|
419
501
|
|
|
420
|
-
names =
|
|
421
|
-
more = deleted.size > 5 ? ", ..." : ""
|
|
502
|
+
names = Text.name_list(deleted)
|
|
422
503
|
verb = deleted.one? ? "subject was" : "subjects were"
|
|
423
|
-
@diagnostics.inform("#{deleted.size} changed #{verb} deleted, nothing to mutate: #{names}
|
|
504
|
+
@diagnostics.inform("#{deleted.size} changed #{verb} deleted, nothing to mutate: #{names}")
|
|
424
505
|
end
|
|
425
506
|
|
|
426
507
|
def report_unchanged_syntax(subjects)
|
|
427
508
|
return if subjects.empty?
|
|
428
509
|
|
|
429
|
-
names =
|
|
430
|
-
more = subjects.size > 5 ? ", ..." : ""
|
|
510
|
+
names = Text.name_list(subjects)
|
|
431
511
|
@diagnostics.inform("#{subjects.size} changed subject(s) differ only in comments or formatting, " \
|
|
432
|
-
"so they are skipped: #{names}
|
|
512
|
+
"so they are skipped: #{names}")
|
|
433
513
|
end
|
|
434
514
|
|
|
435
515
|
def canonical_path(file)
|