saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -1,47 +1,443 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "pathname"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class Engine
|
|
6
|
-
#
|
|
7
|
-
# naming convention (lib/a/b.rb -> spec/a/b_spec.rb)
|
|
8
|
+
# The specs that sit next to the mutated source files by
|
|
9
|
+
# naming convention (lib/a/b.rb -> spec/a/b_spec.rb), or the explicit
|
|
10
|
+
# mapping from config/saboteur_custom_map.rb for layouts the convention
|
|
11
|
+
# cannot guess (#369). A guess, not a
|
|
8
12
|
# coverage fact: a spec that only reaches the file through another one
|
|
9
13
|
# is not found. Packwerk packs (a directory with a package.yml) map to the
|
|
10
14
|
# spec directory inside the same pack, and controllers also map to request
|
|
11
|
-
# specs.
|
|
15
|
+
# specs. With --since, only files whose changed lines carry a mutation
|
|
16
|
+
# subject need a guessed spec; without --since, every included file with
|
|
17
|
+
# a mutation subject needs one. A file that needs a spec and has none
|
|
18
|
+
# stops the run before boot.
|
|
12
19
|
class TestGuess
|
|
13
|
-
|
|
20
|
+
# hints: unmatched file -> its conventional spec path (which does
|
|
21
|
+
# not exist; that is why the file is unmatched), for the stop-before-boot
|
|
22
|
+
# error. The result also builds its own messages, so a stubbed guess in
|
|
23
|
+
# the engine's specs formats exactly like a real one.
|
|
24
|
+
# changed_specs: spec files the diff adds or changes that run, already
|
|
25
|
+
# in specs; skipped_specs: spec files the diff touches that cannot run,
|
|
26
|
+
# each with the reason.
|
|
27
|
+
Result = Data.define(:specs, :unmatched, :subjectless, :changed, :hints, :changed_specs,
|
|
28
|
+
:skipped_specs, :out_of_scope, :excluded, :near) do
|
|
29
|
+
def initialize(changed_specs: [], skipped_specs: {}, out_of_scope: [], excluded: [], near: {}, **fields)
|
|
30
|
+
super
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The stop-before-boot usage error: names each expected spec and the
|
|
34
|
+
# ways out as an append-only instruction - never a reconstructed
|
|
35
|
+
# command, which could carry a hostile ref or a wrong source scope.
|
|
36
|
+
# When the diff deleted the expected spec or added others, it says
|
|
37
|
+
# so: a renamed spec is the usual cause.
|
|
38
|
+
def unmatched_message
|
|
39
|
+
pairs = unmatched.map { |file| pair_description(file) }
|
|
40
|
+
changed = changed_specs.any? ? " Specs the diff adds or changes: #{changed_specs.join(', ')}." : ""
|
|
41
|
+
"no spec found for #{pairs.join(', ')}, so nothing ran.#{changed} Re-run the same command with " \
|
|
42
|
+
"one --test per existing spec path, or append --all to use the whole suite without " \
|
|
43
|
+
"changing the source scope."
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def pair_description(file)
|
|
49
|
+
expected = hints.fetch(file)
|
|
50
|
+
gone = skipped_specs[expected] == DELETED ? ", deleted in the diff" : ""
|
|
51
|
+
found = near.fetch(file, []).any? ? "; found: #{near.fetch(file).join(', ')}" : ""
|
|
52
|
+
"#{file} (expected #{expected}#{gone}#{found})"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
public
|
|
56
|
+
|
|
57
|
+
# The spec files the diff touched that do not run, with the reasons.
|
|
58
|
+
def skipped_message
|
|
59
|
+
named = skipped_specs.map { |path, reason| "#{path} (#{reason})" }
|
|
60
|
+
"inferred specs: not running spec files changed in the diff: #{named.join(', ')}"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Why the subjectless files needed no spec.
|
|
64
|
+
def subjectless_message(since:)
|
|
65
|
+
detail = since ? "no changed mutation subjects" : "no mutation subjects"
|
|
66
|
+
"inferred specs: no spec is required for #{subjectless.join(', ')}: they have #{detail}"
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# The per-file warning when the run proceeds with the mapped
|
|
70
|
+
# subset (#370): the file's mutants run only against the inferred
|
|
71
|
+
# specs, so lines nothing else covers surface as NoCoverage.
|
|
72
|
+
def unmatched_warning(file)
|
|
73
|
+
"inferred specs: no spec found for #{pair_description(file)}; its mutants run only " \
|
|
74
|
+
"against the inferred specs and a line they miss is reported as NoCoverage"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The stop for a --since run whose changed source files all sit outside the
|
|
78
|
+
# include scope (#483): falling back to the whole suite would run the slowest
|
|
79
|
+
# possible suite against zero subjects, so the run refuses and names what the
|
|
80
|
+
# scope hid. Append-only recovery, like unmatched_message: extend include, or
|
|
81
|
+
# name the specs with --test.
|
|
82
|
+
def out_of_scope_message
|
|
83
|
+
named = out_of_scope.first(10).join(", ")
|
|
84
|
+
more = out_of_scope.size - 10
|
|
85
|
+
named += " and #{more} more" if more.positive?
|
|
86
|
+
"no changed source file lies inside the include scope, so nothing ran. " \
|
|
87
|
+
"Changed source files outside it: #{named}. Extend include to cover them, " \
|
|
88
|
+
"or re-run the same command with one --test per existing spec path."
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Why an empty guess mutates nothing rather than widening the run
|
|
92
|
+
# to the whole suite (#369).
|
|
93
|
+
def no_specs_message(since:)
|
|
94
|
+
return "#{excluded_message}, so there is nothing to mutate" if excluded.any?
|
|
95
|
+
|
|
96
|
+
if since
|
|
97
|
+
"inferred specs: no changed file has a mutation subject on its changed lines, " \
|
|
98
|
+
"so there is nothing to mutate"
|
|
99
|
+
else
|
|
100
|
+
"inferred specs: no source file has a mutation subject, so there is nothing to mutate"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Why the custom-map-excluded files needed no spec (#369).
|
|
105
|
+
def excluded_message
|
|
106
|
+
"inferred specs: no spec is required for #{excluded.join(', ')}: " \
|
|
107
|
+
"intentionally untested in config/saboteur_custom_map.rb"
|
|
108
|
+
end
|
|
14
109
|
|
|
110
|
+
# The warning for a guess-scoped run: the guessed specs now stand as
|
|
111
|
+
# explicit tests, with the cross-file coverage caveat that brings.
|
|
112
|
+
def scope_message(since:)
|
|
113
|
+
who = since ? SINCE_CAVEAT_SUBJECT : CAVEAT_SUBJECT
|
|
114
|
+
added = changed_specs.any? ? " (changed in the diff: #{changed_specs.join(', ')})" : ""
|
|
115
|
+
"inferred specs: running only #{specs.join(', ')}#{added}. #{who} only through another file " \
|
|
116
|
+
"does not run, so a mutant only it would kill can survive, and a line no inferred spec " \
|
|
117
|
+
"executes is reported as NoCoverage"
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
CAVEAT_SUBJECT = "A spec that reaches a source file"
|
|
122
|
+
SINCE_CAVEAT_SUBJECT = "A spec the diff did not change that reaches a changed file"
|
|
15
123
|
SOURCE_ROOTS = %w[lib app].freeze
|
|
124
|
+
DELETED = "deleted in the diff"
|
|
125
|
+
OUTSIDE_SUITE = "not loaded by the configured RSpec suite: its pattern, exclude pattern or default path " \
|
|
126
|
+
"leaves it out"
|
|
127
|
+
NOT_REGULAR = "not a regular file"
|
|
16
128
|
|
|
17
|
-
def initialize(config)
|
|
18
|
-
@
|
|
129
|
+
def initialize(config, diagnostics: nil, custom_map: nil)
|
|
130
|
+
@diagnostics = diagnostics
|
|
131
|
+
# Retagged losslessly like Config#root, so a join with a UTF-8 git
|
|
132
|
+
# path never mixes encodings whatever the locale tagged it with.
|
|
133
|
+
@root = Text.utf8_identity(File.realpath(config.root))
|
|
19
134
|
@config = config
|
|
135
|
+
@custom_map = custom_map || CustomMap.load(config.path("custom_map"), root: config.root)
|
|
20
136
|
end
|
|
21
137
|
|
|
22
|
-
# specs: existing spec files (relative paths); unmatched: changed
|
|
23
|
-
# source files with no guessed spec; changed: how many source files
|
|
24
|
-
# changed.
|
|
25
138
|
def call
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
139
|
+
@config["since"] ? call_diff : call_all
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Subject-scoped inference (--subject runs): only the named subjects'
|
|
143
|
+
# files need a guessed spec; every one of them has a subject by
|
|
144
|
+
# construction, so there is no subjectless bucket. With --since the
|
|
145
|
+
# diff narrows the scope further: a named subject whose file is
|
|
146
|
+
# unchanged, whose changed lines sit in other methods, or whose change
|
|
147
|
+
# is comment or formatting only is out of scope and needs no spec.
|
|
148
|
+
def call_for(files, patterns = [])
|
|
149
|
+
files = effective_subject_files(files, patterns) if @config["since"]
|
|
150
|
+
guesses = files.to_h { |file| [file, guess_for(file)] }
|
|
151
|
+
result(guesses, [], files.size)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# specs: existing spec files (relative paths) guessed for the files in
|
|
155
|
+
# scope, plus every spec file the diff adds or changes (#307): a pull
|
|
156
|
+
# request often tests a change through a request or controller spec
|
|
157
|
+
# whose name maps to no changed file. unmatched: such files with no
|
|
158
|
+
# guessed spec; subjectless: files with no mutation subject the run
|
|
159
|
+
# checks, which neither need nor get a spec; changed: how many source
|
|
160
|
+
# files are in scope; hints: each unmatched file's conventional spec
|
|
161
|
+
# path.
|
|
162
|
+
def call_diff
|
|
163
|
+
lines_by_file = changed_lines_by_file
|
|
164
|
+
subjectless = subjectless_files(lines_by_file)
|
|
165
|
+
# Only files with a changed mutation subject get a guessed spec: a
|
|
166
|
+
# subjectless file mutates nothing, so its spec would only narrow the
|
|
167
|
+
# run for no reason.
|
|
168
|
+
guesses = lines_by_file.except(*subjectless).to_h { |file| [file, guess_for(file)] }
|
|
169
|
+
# Nothing in scope: name the changed source the include hid (#483).
|
|
170
|
+
out_of_scope = lines_by_file.empty? ? out_of_scope_source_files : []
|
|
171
|
+
result(guesses, subjectless, lines_by_file.size, out_of_scope: out_of_scope)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Without --since every included file is in scope: a file needs its
|
|
175
|
+
# guessed spec unless it holds no mutation subject at all.
|
|
176
|
+
def call_all
|
|
177
|
+
finder = SubjectFinder.new
|
|
178
|
+
guesses = {}
|
|
179
|
+
subjectless = []
|
|
180
|
+
included_files.each do |file|
|
|
181
|
+
if finder.call(File.join(@root, file)).subjects.empty?
|
|
182
|
+
subjectless << file
|
|
183
|
+
else
|
|
184
|
+
guesses[file] = guess_for(file)
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
result(guesses, subjectless.sort, guesses.size + subjectless.size)
|
|
30
188
|
end
|
|
31
189
|
|
|
32
190
|
private
|
|
33
191
|
|
|
34
|
-
#
|
|
35
|
-
|
|
192
|
+
# The named subjects' files inside the diff scope: the subject must
|
|
193
|
+
# sit on a changed line and survive the comment/formatting prune,
|
|
194
|
+
# mirroring the run's subject selection.
|
|
195
|
+
def effective_subject_files(files, patterns)
|
|
196
|
+
lines_by_file = changed_lines_by_file
|
|
197
|
+
comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(@config["since"], root: @root), root: @root)
|
|
198
|
+
finder = SubjectFinder.new
|
|
199
|
+
constant_finder = ConstantSubjectFinder.new
|
|
200
|
+
files.select do |file|
|
|
201
|
+
lines = lines_by_file[file]
|
|
202
|
+
next false unless lines
|
|
203
|
+
|
|
204
|
+
subjects_in(finder, constant_finder, file).any? do |subject|
|
|
205
|
+
patterns.any? { |pattern| SubjectSelection.match?(subject.id, pattern) } &&
|
|
206
|
+
subject.overlaps_lines?(lines) &&
|
|
207
|
+
!comparison.unchanged?(subject)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
# Changed specs join only a run that mutates something: with no file
|
|
213
|
+
# in scope there is nothing for them to kill.
|
|
214
|
+
def result(guesses, subjectless, changed, out_of_scope: [])
|
|
215
|
+
unmatched = guesses.select { |_, specs| specs.nil? }.keys
|
|
216
|
+
excluded = guesses.select { |_, specs| specs == :excluded }.keys.sort
|
|
217
|
+
inferred = guesses.values.grep(Array).flatten
|
|
218
|
+
diff = @config["since"] && guesses.any? ? specs_in_diff(inferred) : { run: [], skipped: {} }
|
|
219
|
+
unmatched -= covered_by_changed_specs(unmatched, diff[:run])
|
|
220
|
+
Result.new(specs: (inferred + diff[:run]).uniq.sort,
|
|
221
|
+
unmatched: unmatched,
|
|
222
|
+
subjectless: subjectless,
|
|
223
|
+
changed: changed,
|
|
224
|
+
hints: unmatched.to_h { |file| [file, expected_spec(file, diff[:skipped])] },
|
|
225
|
+
near: unmatched.to_h { |file| [file, nearby_specs(file)] },
|
|
226
|
+
changed_specs: diff[:run],
|
|
227
|
+
skipped_specs: diff[:skipped],
|
|
228
|
+
out_of_scope: out_of_scope,
|
|
229
|
+
excluded: excluded)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Changed Ruby files the include scope hides (#483), relative to the
|
|
233
|
+
# root: the diff touched source, but none of it lands inside include.
|
|
234
|
+
# A changed spec is a test, not hidden source; a spec-only diff keeps
|
|
235
|
+
# the whole-suite fallback it always had.
|
|
236
|
+
def out_of_scope_source_files
|
|
237
|
+
included = included_files.to_set
|
|
238
|
+
Selection::GitDiff.changed_paths(@config["since"], root: @root)[:present]
|
|
239
|
+
.select { |path| path.end_with?(".rb") && !spec?(path) && !included.include?(path) }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Spec files under the root that the diff since the base adds, changes
|
|
243
|
+
# or deletes. A changed file runs when the configured suite loads it
|
|
244
|
+
# and it is a regular file; RSpec applies its pattern only to the
|
|
245
|
+
# directories it scans, never to a file named as a test, so the suite's
|
|
246
|
+
# own list decides (a fixture under an excluded directory must not
|
|
247
|
+
# run). A changed *_spec.rb that cannot run is named with the reason.
|
|
248
|
+
# An inferred spec is left to inference, which runs it as before
|
|
249
|
+
# #307, so it is neither added again nor reported as skipped.
|
|
250
|
+
def specs_in_diff(inferred)
|
|
251
|
+
paths = Selection::GitDiff.changed_paths(@config["since"], root: @root)
|
|
252
|
+
skipped = paths[:deleted].select { |path| spec?(path) }.to_h { |path| [path, DELETED] }
|
|
253
|
+
candidates = paths[:present] - inferred
|
|
254
|
+
run = candidates.select { |path| spec?(path) || in_suite?(path) }.select do |path|
|
|
255
|
+
reason = unrunnable_reason(path)
|
|
256
|
+
skipped[path] = reason if reason
|
|
257
|
+
reason.nil?
|
|
258
|
+
end
|
|
259
|
+
{ run: run, skipped: skipped.sort.to_h }
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# A consolidated spec (claims_spec.rb style) never matches the
|
|
263
|
+
# per-file guess, but when the diff itself changes that spec and the
|
|
264
|
+
# spec reads a constant the unmatched file defines, the file is
|
|
265
|
+
# covered: the spec already runs via changed_specs, so stopping would
|
|
266
|
+
# block a run that had something to kill (#538). A mutable change no
|
|
267
|
+
# changed spec exercises still stops.
|
|
268
|
+
def covered_by_changed_specs(unmatched, changed_specs)
|
|
269
|
+
return [] if unmatched.empty? || changed_specs.empty?
|
|
270
|
+
|
|
271
|
+
specs = changed_specs.map { |path| File.join(@root, path) }
|
|
272
|
+
unmatched.select do |file|
|
|
273
|
+
defined_constants(file).any? { |constant| ConstantReads.across(specs, constant).any? }
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# The constants a file defines: owners of its def subjects
|
|
278
|
+
# ("JWT::Claims::DecodeVerifier" from "...#call") and its constant
|
|
279
|
+
# literal subjects. Shaped for ConstantReads, which matches a bare
|
|
280
|
+
# read under the owner's scope or the constant's full path.
|
|
281
|
+
DefinedConstant = Data.define(:name, :owner, :id)
|
|
282
|
+
|
|
283
|
+
def defined_constants(file)
|
|
284
|
+
path = File.join(@root, file)
|
|
285
|
+
owners = SubjectFinder.new.call(path).subjects.filter_map { |subject| subject.id.split(/[#.]/).first }
|
|
286
|
+
literals = ConstantSubjectFinder.new.call(path).subjects.map(&:id)
|
|
287
|
+
(owners + literals).uniq.map do |id|
|
|
288
|
+
owner, _, name = id.rpartition("::")
|
|
289
|
+
DefinedConstant.new(name: name.to_sym, owner: owner.empty? ? nil : owner, id: id)
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# The spec the stop names for an unmatched file: a candidate the diff
|
|
294
|
+
# deleted (a renamed request spec is not always the first), else the
|
|
295
|
+
# most specific one.
|
|
296
|
+
def expected_spec(file, skipped)
|
|
297
|
+
candidates = candidates_for(file)
|
|
298
|
+
candidates.find { |path| skipped[path] == DELETED } || candidates.first
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# Existing spec files with the same basename anywhere under the spec
|
|
302
|
+
# root: the stop and the warning point at them so the user can
|
|
303
|
+
# --test without hunting (#370). Bounded; computed only for
|
|
304
|
+
# unmatched files, which are about to stop or warn.
|
|
305
|
+
def nearby_specs(file)
|
|
306
|
+
# The basename is data, not a pattern: [, ], {, }, ?, * and \ are
|
|
307
|
+
# glob metacharacters and must be escaped to match literally.
|
|
308
|
+
basename = File.basename(file, ".rb").gsub(/[\[\]{}?*\\]/) { |char| "\\#{char}" }
|
|
309
|
+
Dir.glob("spec/**/#{basename}_spec.rb", base: @root)
|
|
310
|
+
.select { |path| File.file?(File.join(@root, path)) }.sort.first(3)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# lstat, not stat: a symlink can point anywhere, including a spec of
|
|
314
|
+
# another application or a fixture the suite excludes.
|
|
315
|
+
def unrunnable_reason(path)
|
|
316
|
+
return NOT_REGULAR unless File.lstat(File.join(@root, path)).file?
|
|
317
|
+
return @suite_error if suite_files.nil?
|
|
318
|
+
|
|
319
|
+
OUTSIDE_SUITE unless in_suite?(path)
|
|
320
|
+
rescue SystemCallError
|
|
321
|
+
NOT_REGULAR
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# macOS git reports a precomposed (NFC) name, while the directory
|
|
325
|
+
# entry RSpec lists can be decomposed (NFD): the same spec spelled two
|
|
326
|
+
# ways. Valid UTF-8 compares in NFC (unicode_normalize's default);
|
|
327
|
+
# other bytes compare as they are.
|
|
328
|
+
def in_suite?(path)
|
|
329
|
+
suite = suite_files
|
|
330
|
+
suite ? suite.include?(comparable(path)) : false
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def comparable(path)
|
|
334
|
+
path.valid_encoding? ? path.unicode_normalize : path
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# nil when RSpec cannot list the suite: under a non-UTF-8 locale its
|
|
338
|
+
# own path expansion can mix encodings for a non-ASCII root. A changed
|
|
339
|
+
# spec is then not run, never guessed into the run.
|
|
340
|
+
def suite_files
|
|
341
|
+
return @suite_files if defined?(@suite_files)
|
|
342
|
+
|
|
343
|
+
listed = Integration.fetch(@config["integration"]).new(@config).configured_spec_files(@root)
|
|
344
|
+
@suite_files = listed.to_set { |path| comparable(path) }
|
|
345
|
+
rescue EncodingError => e
|
|
346
|
+
@suite_error = "the configured RSpec suite's files could not be listed (#{e.class})"
|
|
347
|
+
@suite_files = nil
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def spec?(path)
|
|
351
|
+
path.end_with?("_spec.rb")
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Included files relative to the root, sorted for stable output.
|
|
355
|
+
def included_files
|
|
356
|
+
prefix = "#{@root}/"
|
|
357
|
+
@config.included_files.filter_map { |file| file.delete_prefix(prefix) if file.start_with?(prefix) }.sort
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
# Changed files that saboteur would mutate, relative to the root, with
|
|
361
|
+
# their changed line numbers.
|
|
362
|
+
def changed_lines_by_file
|
|
36
363
|
included = @config.included_files.to_set
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
364
|
+
Selection::GitDiff.changed_lines(@config["since"], root: @root, include: included,
|
|
365
|
+
diagnostics: @diagnostics)
|
|
366
|
+
.each_with_object({}) do |(file, lines), map|
|
|
367
|
+
canonical = Selection::CoverageMap.canonical(file)
|
|
368
|
+
map[canonical.delete_prefix("#{@root}/")] = lines if included.include?(canonical)
|
|
369
|
+
end.sort.to_h
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Changed files whose changed lines carry no mutation subject: the run
|
|
373
|
+
# mutates nothing in them, so they cannot require a spec. Mirrors the
|
|
374
|
+
# run's subject selection, including its comment/formatting-only skip:
|
|
375
|
+
# a deletion whose remaining context lines are blank or comments, or a
|
|
376
|
+
# comment-only change inside a method, leaves the file subjectless.
|
|
377
|
+
def subjectless_files(lines_by_file)
|
|
378
|
+
comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(@config["since"], root: @root), root: @root)
|
|
379
|
+
finder = SubjectFinder.new
|
|
380
|
+
constant_finder = ConstantSubjectFinder.new
|
|
381
|
+
# A file whose diff only deletes lines has no new code to mutate:
|
|
382
|
+
# the deletion is the change the suite itself covers. Its marked
|
|
383
|
+
# neighbor lines would otherwise demand a spec the file never had
|
|
384
|
+
# (#484). Subject selection reads the same list, so both layers
|
|
385
|
+
# agree: no subject, no mutant, no spec demand. The list is
|
|
386
|
+
# canonical-absolute; the line map's keys are root-relative, so
|
|
387
|
+
# compare in absolute space (a monorepo root below the git
|
|
388
|
+
# toplevel must not silently disable this).
|
|
389
|
+
deletion_only = Selection::GitDiff.deletion_only_files(@config["since"], root: @root)
|
|
390
|
+
lines_by_file.reject do |file, lines|
|
|
391
|
+
# (reject keeps the files WITH a changed subject; subjectless
|
|
392
|
+
# is what remains)
|
|
393
|
+
next false if deletion_only.include?(File.join(@root, file))
|
|
394
|
+
|
|
395
|
+
subjects_in(finder, constant_finder, file)
|
|
396
|
+
.select { |subject| subject.overlaps_lines?(lines) }
|
|
397
|
+
.any? { |subject| !comparison.unchanged?(subject) }
|
|
398
|
+
end.keys
|
|
41
399
|
end
|
|
42
400
|
|
|
401
|
+
# Def subjects and changed constant literal subjects (#306): both
|
|
402
|
+
# need a spec to run against. Only the --since route asks.
|
|
403
|
+
def subjects_in(finder, constant_finder, file)
|
|
404
|
+
path = File.join(@root, file)
|
|
405
|
+
finder.call(path).subjects + constant_finder.call(path).subjects
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
# The custom map wins over the naming convention: it exists for the
|
|
409
|
+
# layouts the convention cannot guess (#369). Its entries come back as
|
|
410
|
+
# an array of spec paths, or :excluded for a file the map marks
|
|
411
|
+
# intentionally untested - such a file needs no guessed spec, so it is
|
|
412
|
+
# neither unmatched nor given one. Returns the guessed spec paths, nil
|
|
413
|
+
# when neither the map nor the convention names an existing spec.
|
|
43
414
|
def guess_for(file)
|
|
44
|
-
|
|
415
|
+
mapped = @custom_map.specs_for(file)
|
|
416
|
+
return :excluded if mapped == :excluded
|
|
417
|
+
return expand_mapped(file, mapped) if mapped
|
|
418
|
+
|
|
419
|
+
conventional = candidates_for(file).find { |path| File.file?(File.join(@root, path)) }
|
|
420
|
+
conventional && [conventional]
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
# A mapped spec can be a glob (expanded against the root) or a regexp
|
|
424
|
+
# backreference filled at match time, and neither is validated to
|
|
425
|
+
# exist when the map loads. Only specs that exist run; the literal
|
|
426
|
+
# glob string never reaches RSpec. When the winning entry's specs all
|
|
427
|
+
# vanish the mapping is broken: stop naming it rather than silently
|
|
428
|
+
# guessing on and running the wrong specs (#369).
|
|
429
|
+
def expand_mapped(file, specs)
|
|
430
|
+
expanded = specs.flat_map do |spec|
|
|
431
|
+
if spec.match?(/[*?\[{]/)
|
|
432
|
+
Dir.glob(File.join(@root, spec)).map { |path| path.delete_prefix("#{@root}/") }
|
|
433
|
+
else
|
|
434
|
+
[spec]
|
|
435
|
+
end
|
|
436
|
+
end
|
|
437
|
+
expanded = expanded.select { |path| File.file?(File.join(@root, path)) }
|
|
438
|
+
return expanded unless expanded.empty?
|
|
439
|
+
|
|
440
|
+
raise ConfigError, "custom map entry for #{file} names spec(s) #{specs.join(', ')}, which match no file"
|
|
45
441
|
end
|
|
46
442
|
|
|
47
443
|
# Candidate specs for a source file, most specific first: the nearest
|
|
@@ -57,9 +453,27 @@ module Saboteur
|
|
|
57
453
|
rooted = rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
|
|
58
454
|
names = rooted ? [rest.last, "lib/#{rest.last}", stem] : [stem]
|
|
59
455
|
names = names.flat_map { |name| [name, *request_names(name)] }.uniq
|
|
456
|
+
# A dashed gem name keeps dashes under lib/ but its specs live under
|
|
457
|
+
# the underscored require name: lib/acts-as-taggable-on/x.rb tests in
|
|
458
|
+
# spec/acts_as_taggable_on/x_spec.rb. The literal name still wins
|
|
459
|
+
# when both exist (#383).
|
|
460
|
+
names = names.flat_map { |name| [name, name.tr("-", "_")] }.uniq
|
|
461
|
+
names = layout_variants(names) if rooted
|
|
60
462
|
names.map { |name| "#{spec_root}/#{name}_spec.rb" }
|
|
61
463
|
end
|
|
62
464
|
|
|
465
|
+
# Layouts that mirror less of the source path, after every mirrored
|
|
466
|
+
# guess (#394): a spec/unit/ prefix (shoulda-matchers mirrors below
|
|
467
|
+
# it, #393), the basename at the spec root (pundit), and both
|
|
468
|
+
# together (aasm, #370). Weaker guesses come last: the first
|
|
469
|
+
# candidate that exists wins. A file outside the source roots
|
|
470
|
+
# keeps its literal guess only: dropping directories from a path
|
|
471
|
+
# the roots never anchored would guess any same-named spec.
|
|
472
|
+
def layout_variants(names)
|
|
473
|
+
flat = names.map { |name| name.split("/").last }.uniq
|
|
474
|
+
(names + names.map { |name| "unit/#{name}" } + flat + flat.map { |name| "unit/#{name}" }).uniq
|
|
475
|
+
end
|
|
476
|
+
|
|
63
477
|
# Rails tests controllers as request specs: spec/requests/users_spec.rb
|
|
64
478
|
# or spec/requests/users_controller_spec.rb for controllers/users_controller.rb.
|
|
65
479
|
def request_names(name)
|
|
@@ -71,14 +485,13 @@ module Saboteur
|
|
|
71
485
|
|
|
72
486
|
# The nearest ancestor directory holding a package.yml (a packwerk
|
|
73
487
|
# pack), relative to the root. The root itself is not a pack here:
|
|
74
|
-
#
|
|
488
|
+
# Pathname#ascend stops above "." for these relative paths.
|
|
75
489
|
def pack_of(file)
|
|
76
|
-
|
|
77
|
-
|
|
490
|
+
Pathname(file).ascend do |path|
|
|
491
|
+
dir = path.to_s
|
|
78
492
|
return dir if pack_dirs.include?(dir)
|
|
79
|
-
|
|
80
|
-
dir = File.dirname(dir)
|
|
81
493
|
end
|
|
494
|
+
nil
|
|
82
495
|
end
|
|
83
496
|
|
|
84
497
|
def pack_dirs
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Threads do not survive fork, so a run names the ones alive when the
|
|
7
|
+
# workers fork: by the name the owner gave the thread, else the first
|
|
8
|
+
# frame of its backtrace, so the user can tell whether they matter (a
|
|
9
|
+
# connection pool reaper, Sidekiq, an SDK's background flusher).
|
|
10
|
+
# Known-harmless threads say so inline; an unknown thread gets the
|
|
11
|
+
# generic caveat (#263).
|
|
12
|
+
module ThreadWarnings
|
|
13
|
+
# Well-known fork-time threads, with the condition each assurance
|
|
14
|
+
# hangs on. Evidence: rails-mode children reconnect the database
|
|
15
|
+
# (lib/saboteur/frameworks/rails.rb, after_fork); whether the debug
|
|
16
|
+
# gem's console also attaches to forked children depends on its
|
|
17
|
+
# fork_mode, whose default is :both, i.e. attach (RUBY_DEBUG_FORK_MODE;
|
|
18
|
+
# ruby/debug's config.rb and SESSION.activate on_fork: true in
|
|
19
|
+
# session.rb).
|
|
20
|
+
KNOWN_FORK_THREADS = {
|
|
21
|
+
"AR Pool Reaper" => "ActiveRecord's connection-pool reaper: expected harmless when children " \
|
|
22
|
+
"reconnect the database after fork (rails mode does) - it only maintains " \
|
|
23
|
+
"the parent's pool",
|
|
24
|
+
"DEBUGGER__::SESSION@server" => "the debug gem's remote-console server: what it does in " \
|
|
25
|
+
"children depends on RUBY_DEBUG_FORK_MODE - with parent no " \
|
|
26
|
+
"debugger runs in children; with child or both (the gem's " \
|
|
27
|
+
"default) the debugger activates in children and can pause or " \
|
|
28
|
+
"interfere with supervised mutation runs. For an unattended " \
|
|
29
|
+
"run, do not start the remote debugger at all: drop " \
|
|
30
|
+
"RUBY_DEBUG_OPEN or require \"debug\" without :open"
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
33
|
+
def self.warn(diagnostics)
|
|
34
|
+
others = Thread.list - [Thread.current]
|
|
35
|
+
return if others.empty?
|
|
36
|
+
|
|
37
|
+
unknown = false
|
|
38
|
+
list = others.map do |thread|
|
|
39
|
+
name = thread.name || thread.backtrace&.first || "unknown"
|
|
40
|
+
explanation = KNOWN_FORK_THREADS[name]
|
|
41
|
+
unknown = true unless explanation
|
|
42
|
+
explanation ? "#{name} (#{explanation})" : name
|
|
43
|
+
end.join(", ")
|
|
44
|
+
message = "#{others.size} other thread(s) alive at fork time; they do not run in children: #{list}"
|
|
45
|
+
if unknown
|
|
46
|
+
message += "; a thread that does not run in a child cannot do background work a mutant's test may " \
|
|
47
|
+
"rely on (a pool reaper, a flusher), so results can differ when one is left behind"
|
|
48
|
+
end
|
|
49
|
+
diagnostics.warn(message)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "prism"
|
|
5
5
|
require_relative "../inline_suppression"
|
|
6
|
+
require_relative "../constant_subject_finder"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
class Engine
|
|
@@ -37,6 +38,18 @@ module Saboteur
|
|
|
37
38
|
self.class.digest(before.node) == self.class.digest(subject.node)
|
|
38
39
|
end
|
|
39
40
|
|
|
41
|
+
# Subjects the diff deleted from a changed file: defined once at the
|
|
42
|
+
# base of the --since ref and gone from the working tree. A method
|
|
43
|
+
# that moved but still exists keeps its id and is not a deletion.
|
|
44
|
+
def deleted_subjects(files)
|
|
45
|
+
files.flat_map do |file|
|
|
46
|
+
next [] unless file.end_with?(".rb") && File.file?(file)
|
|
47
|
+
|
|
48
|
+
current_ids = unique_by_id(File.read(file), file).keys
|
|
49
|
+
base_subjects(file).values.reject { |subject| current_ids.include?(subject.id) }
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
40
53
|
# The node as plain data: type and fields, never locations (except
|
|
41
54
|
# the line a `__LINE__` stands for).
|
|
42
55
|
def self.digest(value)
|
|
@@ -64,9 +77,11 @@ module Saboteur
|
|
|
64
77
|
end
|
|
65
78
|
|
|
66
79
|
# Only the names defined once: with two defs of a name, which one a
|
|
67
|
-
# change belongs to is a guess.
|
|
80
|
+
# change belongs to is a guess. Constant literal subjects compare the
|
|
81
|
+
# same way (#306): a comment-only change to the literal prunes them.
|
|
68
82
|
def unique_by_id(source, file)
|
|
69
|
-
subjects = SubjectFinder.new.call(file, source: source).subjects
|
|
83
|
+
subjects = SubjectFinder.new.call(file, source: source).subjects +
|
|
84
|
+
ConstantSubjectFinder.new.call(file, source: source).subjects
|
|
70
85
|
counts = subjects.map(&:id).tally
|
|
71
86
|
subjects.select { |subject| counts[subject.id] == 1 }.to_h { |subject| [subject.id, subject] }
|
|
72
87
|
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# A whole-suite run (--all, c.test_selection = :all, or a --since diff
|
|
7
|
+
# with nothing to infer from) runs the loaded suite for the baseline
|
|
8
|
+
# checks; a cold coverage cache adds one more pass. Say so - with the
|
|
9
|
+
# example count and the real pass count - before those hours pass; past
|
|
10
|
+
# a size threshold, add the way to scope it (#260). The check count
|
|
11
|
+
# comes from c.baseline_passes (#55).
|
|
12
|
+
module UnscopedSuiteWarning
|
|
13
|
+
# A suite this size pays hours for an unscoped run; below it the cost
|
|
14
|
+
# summary alone is enough, without the --test lecture (#260).
|
|
15
|
+
COSTLY_SUITE_EXAMPLES = 1_000
|
|
16
|
+
REUSED_MAP_NOTE = "; the cached coverage map is reused"
|
|
17
|
+
|
|
18
|
+
def self.call(config:, diagnostics:, integration:, full_passes:)
|
|
19
|
+
return unless config["tests"].empty?
|
|
20
|
+
return unless config["since"] || config["all"] || config["test_selection"] == "all"
|
|
21
|
+
|
|
22
|
+
examples = integration.all_tests.size
|
|
23
|
+
coverage = full_passes == config["baseline_passes"] ? REUSED_MAP_NOTE : " + 1 coverage map build"
|
|
24
|
+
message = "running the loaded suite unscoped: #{examples} examples in #{integration.test_files.size} " \
|
|
25
|
+
"spec files; #{full_passes} full passes (#{checks_label(config)}#{coverage})."
|
|
26
|
+
if examples >= COSTLY_SUITE_EXAMPLES
|
|
27
|
+
message += " Pass --test to run only the specs you name: faster, but a mutant of a changed method is then " \
|
|
28
|
+
"killed only by those specs, so specs elsewhere that call this code are not run. " \
|
|
29
|
+
"The default inferred selection runs the specs named after the mutated files, " \
|
|
30
|
+
"a guess that can miss specs elsewhere."
|
|
31
|
+
end
|
|
32
|
+
diagnostics.warn(message)
|
|
33
|
+
diagnostics.inform(message)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.checks_label(config)
|
|
37
|
+
"#{config['baseline_passes']} baseline check#{'s' unless config['baseline_passes'] == 1}"
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|