minitest-distributed 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/ruby.yml +48 -0
- data/.gitignore +8 -0
- data/.rubocop.yml +63 -0
- data/.travis.yml +6 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +53 -0
- data/LICENSE.txt +21 -0
- data/README.md +115 -0
- data/Rakefile +12 -0
- data/bin/console +15 -0
- data/bin/rake +29 -0
- data/bin/rubocop +29 -0
- data/bin/setup +8 -0
- data/bin/srb +29 -0
- data/lib/minitest/distributed.rb +36 -0
- data/lib/minitest/distributed/configuration.rb +53 -0
- data/lib/minitest/distributed/coordinators/coordinator_interface.rb +29 -0
- data/lib/minitest/distributed/coordinators/memory_coordinator.rb +67 -0
- data/lib/minitest/distributed/coordinators/redis_coordinator.rb +387 -0
- data/lib/minitest/distributed/enqueued_runnable.rb +88 -0
- data/lib/minitest/distributed/filters/exclude_filter.rb +35 -0
- data/lib/minitest/distributed/filters/filter_interface.rb +25 -0
- data/lib/minitest/distributed/filters/include_filter.rb +35 -0
- data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb +76 -0
- data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb +48 -0
- data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb +61 -0
- data/lib/minitest/distributed/result_aggregate.rb +67 -0
- data/lib/minitest/distributed/result_type.rb +28 -0
- data/lib/minitest/distributed/test_runner.rb +37 -0
- data/lib/minitest/distributed/test_selector.rb +54 -0
- data/lib/minitest/distributed/version.rb +8 -0
- data/lib/minitest/distributed_plugin.rb +51 -0
- data/minitest-distributed.gemspec +50 -0
- data/sorbet/config +2 -0
- data/sorbet/rbi/minitest.rbi +238 -0
- data/sorbet/rbi/rbconfig.rbi +6 -0
- data/sorbet/rbi/redis.rbi +70 -0
- data/sorbet/rbi/winsize.rbi +7 -0
- metadata +142 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
# typed: strict
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Minitest
|
5
|
+
module Distributed
|
6
|
+
class TestRunner
|
7
|
+
extend T::Sig
|
8
|
+
|
9
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
10
|
+
attr_reader :options
|
11
|
+
|
12
|
+
sig { returns(Configuration) }
|
13
|
+
attr_reader :configuration
|
14
|
+
|
15
|
+
sig { returns(TestSelector) }
|
16
|
+
attr_reader :test_selector
|
17
|
+
|
18
|
+
sig { returns(Coordinators::CoordinatorInterface) }
|
19
|
+
attr_reader :coordinator
|
20
|
+
|
21
|
+
sig { params(options: T::Hash[Symbol, T.untyped]).void }
|
22
|
+
def initialize(options)
|
23
|
+
@options = options
|
24
|
+
|
25
|
+
@configuration = T.let(@options[:distributed], Configuration)
|
26
|
+
@coordinator = T.let(configuration.coordinator, Coordinators::CoordinatorInterface)
|
27
|
+
@test_selector = T.let(TestSelector.new(options), TestSelector)
|
28
|
+
end
|
29
|
+
|
30
|
+
sig { params(reporter: AbstractReporter).void }
|
31
|
+
def run(reporter)
|
32
|
+
coordinator.produce(test_selector: test_selector)
|
33
|
+
coordinator.consume(reporter: reporter)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# typed: strict
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Minitest
|
5
|
+
module Distributed
|
6
|
+
class TestSelector
|
7
|
+
extend T::Sig
|
8
|
+
|
9
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
10
|
+
attr_reader :options
|
11
|
+
|
12
|
+
sig { returns(T::Array[Filters::FilterInterface]) }
|
13
|
+
attr_reader :filters
|
14
|
+
|
15
|
+
sig { params(options: T::Hash[Symbol, T.untyped]).void }
|
16
|
+
def initialize(options)
|
17
|
+
@options = options
|
18
|
+
|
19
|
+
@filters = T.let([], T::Array[Filters::FilterInterface])
|
20
|
+
initialize_filters
|
21
|
+
end
|
22
|
+
|
23
|
+
sig { void }
|
24
|
+
def initialize_filters
|
25
|
+
@filters << Filters::IncludeFilter.new(options[:filter]) if options[:filter]
|
26
|
+
@filters << Filters::ExcludeFilter.new(options[:exclude]) if options[:exclude]
|
27
|
+
end
|
28
|
+
|
29
|
+
sig { returns(T::Array[EnqueuedRunnable]) }
|
30
|
+
def discover_tests
|
31
|
+
Minitest::Runnable.runnables.flat_map do |runnable|
|
32
|
+
runnable.runnable_methods.map do |method_name|
|
33
|
+
EnqueuedRunnable.new(class_name: runnable.name, method_name: method_name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
sig { params(tests: T::Array[EnqueuedRunnable]).returns(T::Array[EnqueuedRunnable]) }
|
39
|
+
def select_tests(tests)
|
40
|
+
return tests if filters.empty?
|
41
|
+
tests.flat_map do |runnable_method|
|
42
|
+
filters.flat_map do |filter|
|
43
|
+
filter.call(runnable_method)
|
44
|
+
end
|
45
|
+
end.compact
|
46
|
+
end
|
47
|
+
|
48
|
+
sig { returns(T::Array[EnqueuedRunnable]) }
|
49
|
+
def tests
|
50
|
+
select_tests(discover_tests)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# typed: true
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require_relative './distributed'
|
5
|
+
|
6
|
+
module Minitest
|
7
|
+
class << self
|
8
|
+
extend T::Sig
|
9
|
+
|
10
|
+
def plugin_distributed_options(opts, options)
|
11
|
+
options[:distributed] = Minitest::Distributed::Configuration.from_env
|
12
|
+
|
13
|
+
opts.on('--coordinator=URI', "The URI pointing to the coordinator") do |uri|
|
14
|
+
options[:distributed].coordinator_uri = URI.parse(uri)
|
15
|
+
end
|
16
|
+
|
17
|
+
opts.on('--test-timeout=TIMEOUT', "The maximum run time for a single test in seconds") do |timeout|
|
18
|
+
options[:distributed].test_timeout = Integer(timeout)
|
19
|
+
end
|
20
|
+
|
21
|
+
opts.on('--max-attempts=ATTEMPTS', "The maximum number of attempts to run a test") do |attempts|
|
22
|
+
options[:distributed].max_attempts = Integer(attempts)
|
23
|
+
end
|
24
|
+
|
25
|
+
opts.on('--test-batch-size=NUMBER', "The number of tests to process per batch") do |batch_size|
|
26
|
+
options[:distributed].test_batch_size = Integer(batch_size)
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on('--run-id=ID', "The ID for this run shared between coordinated workers") do |id|
|
30
|
+
options[:distributed].run_id = id
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on('--worker-id=ID', "The unique ID for this worker") do |id|
|
34
|
+
options[:distributed].worker_id = id
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def plugin_distributed_init(options)
|
39
|
+
Minitest.singleton_class.prepend(Minitest::Distributed::TestRunnerPatch)
|
40
|
+
options[:distributed].coordinator.register_reporters(reporter: reporter, options: options)
|
41
|
+
|
42
|
+
if reporter.reporters.reject! { |reporter| reporter.is_a?(Minitest::ProgressReporter) }
|
43
|
+
reporter << Minitest::Distributed::Reporters::DistributedPogressReporter.new(options[:io], options)
|
44
|
+
end
|
45
|
+
|
46
|
+
if reporter.reporters.reject! { |reporter| reporter.is_a?(Minitest::SummaryReporter) }
|
47
|
+
reporter << Minitest::Distributed::Reporters::DistributedSummaryReporter.new(options[:io], options)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require_relative 'lib/minitest/distributed/version'
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = "minitest-distributed"
|
5
|
+
spec.version = Minitest::Distributed::VERSION
|
6
|
+
spec.authors = ["Willem van Bergen"]
|
7
|
+
spec.email = ["willem@vanbergen.org"]
|
8
|
+
|
9
|
+
spec.summary = "Distributed test executor plugin for Minitest"
|
10
|
+
spec.description = <<~EOD
|
11
|
+
minitest-distributed is a plugin for minitest for executing tests on a
|
12
|
+
distributed set of unreliable workers.
|
13
|
+
|
14
|
+
When a test suite grows large enough, it inevitable gets too slow to run
|
15
|
+
on a single machine to give timely feedback to developers. This plugins
|
16
|
+
combats this issue by distributing the full test suite to a set of workers.
|
17
|
+
Every worker is a consuming from a single queue, so the tests get evenly
|
18
|
+
distributed and all workers will finish around the same time. Redis is used
|
19
|
+
as coordinator, but when using this plugin without having access to Redis,
|
20
|
+
it will use an in-memory coordinator.
|
21
|
+
|
22
|
+
Using multiple (virtual) machines for a test run is an (additional) source
|
23
|
+
of flakiness. To combat flakiness, minitest-distributed implements resiliency
|
24
|
+
patterns, like re-running a test on a different worker on failure, and
|
25
|
+
a circuit breaker for misbehaving workers.
|
26
|
+
EOD
|
27
|
+
|
28
|
+
spec.homepage = "https://github.com/Shopify/minitest-distributed"
|
29
|
+
spec.license = "MIT"
|
30
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
31
|
+
|
32
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
33
|
+
|
34
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
35
|
+
spec.metadata["source_code_uri"] = "https://github.com/Shopify/minitest-distributed"
|
36
|
+
# spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
|
37
|
+
|
38
|
+
# Specify which files should be added to the gem when it is released.
|
39
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
40
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
41
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
42
|
+
end
|
43
|
+
spec.bindir = "exe"
|
44
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
45
|
+
spec.require_paths = ["lib"]
|
46
|
+
|
47
|
+
spec.add_dependency('minitest', '~> 5.12')
|
48
|
+
spec.add_dependency('redis', '~> 4.1')
|
49
|
+
spec.add_dependency('sorbet-runtime')
|
50
|
+
end
|
data/sorbet/config
ADDED
@@ -0,0 +1,238 @@
|
|
1
|
+
# This file is autogenerated. Do not edit it by hand. Regenerate it with:
|
2
|
+
# srb rbi sorbet-typed
|
3
|
+
#
|
4
|
+
# If you would like to make changes to this file, great! Please upstream any changes you make here:
|
5
|
+
#
|
6
|
+
# https://github.com/sorbet/sorbet-typed/edit/master/lib/minitest/all/minitest.rbi
|
7
|
+
#
|
8
|
+
# typed: strong
|
9
|
+
|
10
|
+
module Minitest
|
11
|
+
class Runnable
|
12
|
+
def self.run_one_method(klass, method_name, reporter); end
|
13
|
+
def self.runnables; end
|
14
|
+
|
15
|
+
def name; end
|
16
|
+
def time; end
|
17
|
+
def time=(duration); end
|
18
|
+
def failures; end
|
19
|
+
|
20
|
+
def initialize(method_name); end
|
21
|
+
end
|
22
|
+
|
23
|
+
class Test < Runnable
|
24
|
+
include Minitest::Assertions
|
25
|
+
end
|
26
|
+
|
27
|
+
class Result < Runnable
|
28
|
+
sig { returns(String) }
|
29
|
+
def name; end
|
30
|
+
|
31
|
+
sig { returns(String) }
|
32
|
+
def klass; end
|
33
|
+
|
34
|
+
sig { returns(String) }
|
35
|
+
def class_name; end
|
36
|
+
|
37
|
+
sig { returns(T.nilable(Minitest::Assertion)) }
|
38
|
+
def failure; end
|
39
|
+
|
40
|
+
sig { returns(T::Boolean) }
|
41
|
+
def error?; end
|
42
|
+
|
43
|
+
sig { returns(T::Boolean) }
|
44
|
+
def skipped?; end
|
45
|
+
|
46
|
+
sig { returns(T::Boolean) }
|
47
|
+
def passed?; end
|
48
|
+
|
49
|
+
sig { returns(Integer) }
|
50
|
+
def assertions; end
|
51
|
+
|
52
|
+
sig { params(runnable: Runnable).returns(T.attached_class) }
|
53
|
+
def self.from(runnable); end
|
54
|
+
end
|
55
|
+
|
56
|
+
class Assertion < Exception
|
57
|
+
sig { returns(String) }
|
58
|
+
def result_label; end
|
59
|
+
|
60
|
+
sig { returns(String) }
|
61
|
+
def result_code; end
|
62
|
+
end
|
63
|
+
|
64
|
+
class Skip < Exception
|
65
|
+
end
|
66
|
+
|
67
|
+
class UnexpectedError < Assertion
|
68
|
+
end
|
69
|
+
|
70
|
+
class AbstractReporter
|
71
|
+
sig { void }
|
72
|
+
def start; end
|
73
|
+
|
74
|
+
sig { params(runnable: T.class_of(Runnable), method_name: String).void }
|
75
|
+
def prerecord(runnable, method_name); end
|
76
|
+
|
77
|
+
sig { params(result: Minitest::Result).void }
|
78
|
+
def record(result); end
|
79
|
+
|
80
|
+
sig { void }
|
81
|
+
def report; end
|
82
|
+
|
83
|
+
sig { returns(T::Boolean) }
|
84
|
+
def passed?; end
|
85
|
+
end
|
86
|
+
|
87
|
+
class Reporter < AbstractReporter
|
88
|
+
sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
|
89
|
+
def initialize(io, options); end
|
90
|
+
|
91
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
92
|
+
def options; end
|
93
|
+
|
94
|
+
sig { returns(IO) }
|
95
|
+
def io; end
|
96
|
+
end
|
97
|
+
|
98
|
+
class StatisticsReporter < Reporter
|
99
|
+
end
|
100
|
+
|
101
|
+
class SummaryReporter < StatisticsReporter
|
102
|
+
end
|
103
|
+
|
104
|
+
class ProgressReporter < Reporter
|
105
|
+
end
|
106
|
+
|
107
|
+
class CompositeReporter < AbstractReporter
|
108
|
+
sig { params(reporter: AbstractReporter).returns(T.self_type) }
|
109
|
+
def <<(reporter); end
|
110
|
+
|
111
|
+
sig { returns(T::Array[AbstractReporter]) }
|
112
|
+
def reporters; end
|
113
|
+
end
|
114
|
+
|
115
|
+
sig { returns(CompositeReporter) }
|
116
|
+
def self.reporter; end
|
117
|
+
|
118
|
+
sig { void }
|
119
|
+
def self.autorun; end
|
120
|
+
|
121
|
+
sig { params(args: T::Array[String]).returns(T::Boolean) }
|
122
|
+
def self.run(args = []); end
|
123
|
+
|
124
|
+
sig { params(klass: T.class_of(Runnable), method_name: String).returns(Minitest::Result) }
|
125
|
+
def self.run_one_method(klass, method_name); end
|
126
|
+
|
127
|
+
sig { returns(Float) }
|
128
|
+
def self.clock_time; end
|
129
|
+
end
|
130
|
+
|
131
|
+
module Minitest::Assertions
|
132
|
+
extend T::Sig
|
133
|
+
|
134
|
+
sig { params(msg: T.nilable(String)).returns(TrueClass) }
|
135
|
+
def pass(msg = nil); end
|
136
|
+
|
137
|
+
sig { params(msg: T.nilable(String)).returns(FalseClass) }
|
138
|
+
def flunk(msg = nil); end
|
139
|
+
|
140
|
+
sig { params(test: T.untyped, msg: T.nilable(String)).returns(TrueClass) }
|
141
|
+
def assert(test, msg = nil); end
|
142
|
+
|
143
|
+
sig do
|
144
|
+
params(
|
145
|
+
exp: BasicObject,
|
146
|
+
msg: T.nilable(String)
|
147
|
+
).returns(TrueClass)
|
148
|
+
end
|
149
|
+
def assert_empty(exp, msg = nil); end
|
150
|
+
|
151
|
+
sig do
|
152
|
+
params(
|
153
|
+
exp: BasicObject,
|
154
|
+
act: BasicObject,
|
155
|
+
msg: T.nilable(String)
|
156
|
+
).returns(TrueClass)
|
157
|
+
end
|
158
|
+
def assert_equal(exp, act, msg = nil); end
|
159
|
+
|
160
|
+
sig do
|
161
|
+
params(
|
162
|
+
collection: T::Enumerable[T.untyped],
|
163
|
+
obj: BasicObject,
|
164
|
+
msg: T.nilable(String)
|
165
|
+
).returns(TrueClass)
|
166
|
+
end
|
167
|
+
def assert_includes(collection, obj, msg = nil); end
|
168
|
+
|
169
|
+
sig do
|
170
|
+
params(
|
171
|
+
obj: BasicObject,
|
172
|
+
msg: T.nilable(String)
|
173
|
+
).returns(TrueClass)
|
174
|
+
end
|
175
|
+
def assert_nil(obj, msg = nil); end
|
176
|
+
|
177
|
+
sig do
|
178
|
+
params(
|
179
|
+
exp: T.untyped
|
180
|
+
).returns(TrueClass)
|
181
|
+
end
|
182
|
+
def assert_raises(*exp); end
|
183
|
+
|
184
|
+
sig do
|
185
|
+
params(
|
186
|
+
obj: BasicObject,
|
187
|
+
predicate: Symbol,
|
188
|
+
msg: T.nilable(String)
|
189
|
+
).returns(TrueClass)
|
190
|
+
end
|
191
|
+
def assert_predicate(obj, predicate, msg = nil); end
|
192
|
+
|
193
|
+
sig { params(test: T.untyped, msg: T.nilable(String)).returns(TrueClass) }
|
194
|
+
def refute(test, msg = nil); end
|
195
|
+
|
196
|
+
sig do
|
197
|
+
params(
|
198
|
+
exp: BasicObject,
|
199
|
+
msg: T.nilable(String)
|
200
|
+
).returns(TrueClass)
|
201
|
+
end
|
202
|
+
def refute_empty(exp, msg = nil); end
|
203
|
+
|
204
|
+
sig do
|
205
|
+
params(
|
206
|
+
exp: BasicObject,
|
207
|
+
act: BasicObject,
|
208
|
+
msg: T.nilable(String)
|
209
|
+
).returns(TrueClass)
|
210
|
+
end
|
211
|
+
def refute_equal(exp, act, msg = nil); end
|
212
|
+
|
213
|
+
sig do
|
214
|
+
params(
|
215
|
+
collection: T::Enumerable[T.untyped],
|
216
|
+
obj: BasicObject,
|
217
|
+
msg: T.nilable(String)
|
218
|
+
).returns(TrueClass)
|
219
|
+
end
|
220
|
+
def refute_includes(collection, obj, msg = nil); end
|
221
|
+
|
222
|
+
sig do
|
223
|
+
params(
|
224
|
+
obj: BasicObject,
|
225
|
+
msg: T.nilable(String)
|
226
|
+
).returns(TrueClass)
|
227
|
+
end
|
228
|
+
def refute_nil(obj, msg = nil); end
|
229
|
+
|
230
|
+
sig do
|
231
|
+
params(
|
232
|
+
obj: BasicObject,
|
233
|
+
predicate: Symbol,
|
234
|
+
msg: T.nilable(String)
|
235
|
+
).returns(TrueClass)
|
236
|
+
end
|
237
|
+
def refute_predicate(obj, predicate, msg = nil); end
|
238
|
+
end
|