minitest-distributed 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/ruby.yml +48 -0
  3. data/.gitignore +8 -0
  4. data/.rubocop.yml +63 -0
  5. data/.travis.yml +6 -0
  6. data/CODE_OF_CONDUCT.md +74 -0
  7. data/Gemfile +12 -0
  8. data/Gemfile.lock +53 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +115 -0
  11. data/Rakefile +12 -0
  12. data/bin/console +15 -0
  13. data/bin/rake +29 -0
  14. data/bin/rubocop +29 -0
  15. data/bin/setup +8 -0
  16. data/bin/srb +29 -0
  17. data/lib/minitest/distributed.rb +36 -0
  18. data/lib/minitest/distributed/configuration.rb +53 -0
  19. data/lib/minitest/distributed/coordinators/coordinator_interface.rb +29 -0
  20. data/lib/minitest/distributed/coordinators/memory_coordinator.rb +67 -0
  21. data/lib/minitest/distributed/coordinators/redis_coordinator.rb +387 -0
  22. data/lib/minitest/distributed/enqueued_runnable.rb +88 -0
  23. data/lib/minitest/distributed/filters/exclude_filter.rb +35 -0
  24. data/lib/minitest/distributed/filters/filter_interface.rb +25 -0
  25. data/lib/minitest/distributed/filters/include_filter.rb +35 -0
  26. data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb +76 -0
  27. data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb +48 -0
  28. data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb +61 -0
  29. data/lib/minitest/distributed/result_aggregate.rb +67 -0
  30. data/lib/minitest/distributed/result_type.rb +28 -0
  31. data/lib/minitest/distributed/test_runner.rb +37 -0
  32. data/lib/minitest/distributed/test_selector.rb +54 -0
  33. data/lib/minitest/distributed/version.rb +8 -0
  34. data/lib/minitest/distributed_plugin.rb +51 -0
  35. data/minitest-distributed.gemspec +50 -0
  36. data/sorbet/config +2 -0
  37. data/sorbet/rbi/minitest.rbi +238 -0
  38. data/sorbet/rbi/rbconfig.rbi +6 -0
  39. data/sorbet/rbi/redis.rbi +70 -0
  40. data/sorbet/rbi/winsize.rbi +7 -0
  41. metadata +142 -0
@@ -0,0 +1,88 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ class EnqueuedRunnable < T::Struct
7
+ class << self
8
+ extend T::Sig
9
+
10
+ sig { params(identifier: String).returns(T.attached_class) }
11
+ def from_identifier(identifier)
12
+ class_name, method_name = identifier.split('#', 2)
13
+ new(
14
+ class_name: T.must(class_name),
15
+ method_name: T.must(method_name),
16
+ )
17
+ end
18
+
19
+ sig { params(runnable: Minitest::Runnable).returns(T.attached_class) }
20
+ def from_runnable(runnable)
21
+ new(
22
+ class_name: T.must(runnable.class.name),
23
+ method_name: runnable.name,
24
+ )
25
+ end
26
+
27
+ sig { params(result: Minitest::Result).returns(T.attached_class) }
28
+ def from_result(result)
29
+ new(class_name: result.class_name, method_name: result.name)
30
+ end
31
+
32
+ sig { params(claims: T::Array[[String, T::Hash[String, String]]]).returns(T::Array[T.attached_class]) }
33
+ def from_redis_stream_claim(claims)
34
+ claims.map do |id, runnable_method_info|
35
+ new(
36
+ class_name: runnable_method_info.fetch('class_name'),
37
+ method_name: runnable_method_info.fetch('method_name'),
38
+ execution_id: id,
39
+ )
40
+ end
41
+ end
42
+
43
+ sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
44
+ def find_runnable_class(name)
45
+ name.split('::')
46
+ .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
47
+ end
48
+ end
49
+
50
+ extend T::Sig
51
+
52
+ const :class_name, String
53
+ const :method_name, String
54
+ const :execution_id, T.nilable(String), dont_store: true
55
+
56
+ # By setting canned failure, we will not actually run the runnable,
57
+ # but immediately return a result with the canned assertion.x
58
+ prop :canned_failure, T.nilable(Minitest::Assertion), dont_store: true
59
+
60
+ sig { returns(String) }
61
+ def identifier
62
+ "#{class_name}##{method_name}"
63
+ end
64
+
65
+ sig { returns(T.class_of(Minitest::Runnable)) }
66
+ def runnable_class
67
+ self.class.find_runnable_class(class_name)
68
+ end
69
+
70
+ sig { returns(Minitest::Runnable) }
71
+ def runnable
72
+ runnable_class.new(method_name)
73
+ end
74
+
75
+ sig { returns(Minitest::Result) }
76
+ def run
77
+ if canned_failure
78
+ canned_runnable = runnable
79
+ canned_runnable.time = 0.0
80
+ canned_runnable.failures << canned_failure
81
+ Minitest::Result.from(canned_runnable)
82
+ else
83
+ Minitest.run_one_method(runnable_class, method_name)
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,35 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ module Filters
7
+ class ExcludeFilter
8
+ extend T::Sig
9
+ include FilterInterface
10
+
11
+ sig { returns(T.any(String, Regexp)) }
12
+ attr_reader :filter
13
+
14
+ sig { params(filter: T.any(String, Regexp)).void }
15
+ def initialize(filter)
16
+ @filter = filter
17
+ if filter.is_a?(String) && (match_info = filter.match(%r%/(.*)/%))
18
+ @filter = Regexp.new(T.must(match_info[1]))
19
+ end
20
+ end
21
+
22
+ sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
23
+ def call(enqueued_runnable)
24
+ # rubocop:disable Style/CaseEquality
25
+ if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
26
+ []
27
+ else
28
+ [enqueued_runnable]
29
+ end
30
+ # rubocop:enable Style/CaseEquality
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,25 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ module Filters
7
+ # A filter proc is a callable object that changes the list of runnables that will
8
+ # be executed during the test run. For every runnable, it should return an
9
+ # array of runnables.
10
+ #
11
+ # - If it returns an empty array, the runnable will not be run.
12
+ # - If it returns a single elemnt array with the passed ion runnable to make no changes.
13
+ # - It can return an array of enumerables to expand the number of runnables in this test run,
14
+ # We use this for grinding tests, for instance.
15
+ module FilterInterface
16
+ extend T::Sig
17
+ extend T::Helpers
18
+ interface!
19
+
20
+ sig { abstract.params(runnable_method: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
21
+ def call(runnable_method); end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,35 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ module Filters
7
+ class IncludeFilter
8
+ extend T::Sig
9
+ include FilterInterface
10
+
11
+ sig { returns(T.any(String, Regexp)) }
12
+ attr_reader :filter
13
+
14
+ sig { params(filter: T.any(String, Regexp)).void }
15
+ def initialize(filter)
16
+ @filter = filter
17
+ if filter.is_a?(String) && (match_info = filter.match(%r%/(.*)/%))
18
+ @filter = Regexp.new(T.must(match_info[1]))
19
+ end
20
+ end
21
+
22
+ sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
23
+ def call(enqueued_runnable)
24
+ # rubocop:disable Style/CaseEquality
25
+ if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
26
+ [enqueued_runnable]
27
+ else
28
+ []
29
+ end
30
+ # rubocop:enable Style/CaseEquality
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,76 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require 'io/console'
5
+
6
+ module Minitest
7
+ module Distributed
8
+ module Reporters
9
+ class DistributedPogressReporter < Minitest::Reporter
10
+ extend T::Sig
11
+
12
+ sig { returns(Coordinators::CoordinatorInterface) }
13
+ attr_reader :coordinator
14
+
15
+ sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
16
+ def initialize(io, options)
17
+ super
18
+ if io.tty?
19
+ io.sync = true
20
+ end
21
+ @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
22
+ end
23
+
24
+ # Note: due to batching and parallel tests, we have no guarantee that `prerecord`
25
+ # and `record` will be called in succession for the same test without calls to
26
+ # either method being interjected for other tests.
27
+ #
28
+ # As a result we have no idea what will be on the last line of the console.
29
+ # We always clear the full line before printing output.
30
+
31
+ sig { override.params(klass: T.class_of(Runnable), name: String).void }
32
+ def prerecord(klass, name)
33
+ if io.tty?
34
+ line_width = clear_current_line
35
+ io.print("[#{results.acks}/#{results.size}] #{klass}##{name}".slice(0...line_width))
36
+ end
37
+ end
38
+
39
+ sig { override.params(result: Minitest::Result).void }
40
+ def record(result)
41
+ clear_current_line if io.tty?
42
+
43
+ case (result_type = ResultType.of(result))
44
+ when ResultType::Passed
45
+ # TODO: warn for tests that are slower than the test timeout.
46
+ when ResultType::Skipped
47
+ io.puts("#{result}\n") if options[:verbose]
48
+ when ResultType::Error, ResultType::Failed
49
+ io.puts("#{result}\n")
50
+ else
51
+ T.absurd(result_type)
52
+ end
53
+ end
54
+
55
+ sig { override.void }
56
+ def report
57
+ clear_current_line if io.tty?
58
+ end
59
+
60
+ private
61
+
62
+ sig { returns(Integer) }
63
+ def clear_current_line
64
+ _height, width = IO.console.winsize
65
+ io.print("\r" + (' ' * width) + "\r")
66
+ width
67
+ end
68
+
69
+ sig { returns(ResultAggregate) }
70
+ def results
71
+ coordinator.combined_results
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,48 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ module Reporters
7
+ class DistributedSummaryReporter < Minitest::Reporter
8
+ extend T::Sig
9
+
10
+ sig { returns(Coordinators::CoordinatorInterface) }
11
+ attr_reader :coordinator
12
+
13
+ sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
14
+ def initialize(io, options)
15
+ super
16
+ io.sync = true
17
+ @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
18
+ @start_time = T.let(0.0, Float)
19
+ end
20
+
21
+ sig { override.void }
22
+ def start
23
+ @start_time = Minitest.clock_time
24
+ io.puts("Run options: #{options[:args]}\n\n")
25
+ end
26
+
27
+ sig { override.void }
28
+ def report
29
+ duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
30
+
31
+ local_results = coordinator.local_results
32
+ combined_results = coordinator.combined_results
33
+ if combined_results == local_results
34
+ io.puts("Results: #{combined_results} #{duration}")
35
+ else
36
+ io.puts("This worker: #{local_results} #{duration}")
37
+ io.puts("Combined results: #{combined_results}")
38
+ end
39
+ end
40
+
41
+ sig { override.returns(T::Boolean) }
42
+ def passed?
43
+ coordinator.combined_results.passed?
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,61 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ module Reporters
7
+ class RedisCoordinatorWarningsReporter < Minitest::Reporter
8
+ extend T::Sig
9
+
10
+ sig { override.void }
11
+ def report
12
+ [reclaim_warning, missing_acks_warning].compact.each do |warning|
13
+ io.puts
14
+ io.puts(warning)
15
+ end
16
+ end
17
+
18
+ private
19
+
20
+ sig { returns(Configuration) }
21
+ def configuration
22
+ options[:distributed]
23
+ end
24
+
25
+ sig { returns(Coordinators::RedisCoordinator) }
26
+ def redis_coordinator
27
+ T.cast(configuration.coordinator, Coordinators::RedisCoordinator)
28
+ end
29
+
30
+ sig { returns(T.nilable(String)) }
31
+ def reclaim_warning
32
+ if redis_coordinator.reclaimed_tests.any?
33
+ <<~WARNING
34
+ WARNING: The following tests were reclaimed from another worker:
35
+ #{redis_coordinator.reclaimed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
36
+
37
+ The original worker did not complete running this test in #{configuration.test_timeout}ms.
38
+ This either means that the worker unexpectedly went away, or that the test is too slow.
39
+ WARNING
40
+ end
41
+ end
42
+
43
+ sig { returns(T.nilable(String)) }
44
+ def missing_acks_warning
45
+ local_results = redis_coordinator.local_results
46
+ if local_results.acks < local_results.size
47
+ <<~WARNING
48
+ WARNING: This worker was not able to ack all the test it ran with the coordinator (#{local_results.acks}/#{local_results.size}).
49
+
50
+ This means that this worker took too long to report the status of one or more tests,
51
+ and these tests were claimed by other workers. As a result, the total number of
52
+ reported runs may be larger than the size of the test suite.
53
+
54
+ Make sure that all your tests complete within #{configuration.test_timeout}ms.
55
+ WARNING
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,67 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ class ResultAggregate < T::Struct
7
+ extend T::Sig
8
+
9
+ prop :runs, Integer, default: 0
10
+ prop :assertions, Integer, default: 0
11
+ prop :passes, Integer, default: 0
12
+ prop :failures, Integer, default: 0
13
+ prop :errors, Integer, default: 0
14
+ prop :skips, Integer, default: 0
15
+ prop :reruns, Integer, default: 0
16
+ prop :acks, Integer, default: 0
17
+ prop :size, Integer, default: 0
18
+
19
+ sig { params(result: Minitest::Result).void }
20
+ def update_with_result(result)
21
+ case (result_type = ResultType.of(result))
22
+ when ResultType::Passed then self.passes += 1
23
+ when ResultType::Failed then self.failures += 1
24
+ when ResultType::Error then self.errors += 1
25
+ when ResultType::Skipped then self.skips += 1
26
+ else T.absurd(result_type)
27
+ end
28
+
29
+ self.runs += 1
30
+ self.assertions += result.assertions
31
+ end
32
+
33
+ sig { returns(String) }
34
+ def to_s
35
+ str = +"#{runs} runs, #{assertions} assertions, #{passes} passes, #{failures} failures, #{errors} errors"
36
+ str << ", #{skips} skips" if skips > 0
37
+ str << ", #{reruns} re-runs" if reruns > 0
38
+ str
39
+ end
40
+
41
+ sig { returns(Integer) }
42
+ def unique_runs
43
+ runs - reruns
44
+ end
45
+
46
+ sig { returns(Integer) }
47
+ def reported_results
48
+ passes + failures + errors + skips
49
+ end
50
+
51
+ sig { returns(T::Boolean) }
52
+ def completed?
53
+ acks == size
54
+ end
55
+
56
+ sig { returns(T::Boolean) }
57
+ def valid?
58
+ unique_runs == reported_results
59
+ end
60
+
61
+ sig { returns(T::Boolean) }
62
+ def passed?
63
+ completed? && valid? && self.failures == 0 && self.errors == 0
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,28 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ module Minitest
5
+ module Distributed
6
+ class ResultType < T::Enum
7
+ enums do
8
+ Passed = new
9
+ Failed = new
10
+ Error = new
11
+ Skipped = new
12
+ end
13
+
14
+ sig { params(result: Minitest::Result).returns(ResultType) }
15
+ def self.of(result)
16
+ if result.passed?
17
+ Passed
18
+ elsif result.error?
19
+ Error
20
+ elsif result.skipped?
21
+ Skipped
22
+ else
23
+ Failed
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end