minitest-distributed 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/ruby.yml +48 -0
  3. data/.gitignore +8 -0
  4. data/.rubocop.yml +63 -0
  5. data/.travis.yml +6 -0
  6. data/CODE_OF_CONDUCT.md +74 -0
  7. data/Gemfile +12 -0
  8. data/Gemfile.lock +53 -0
  9. data/LICENSE.txt +21 -0
  10. data/README.md +115 -0
  11. data/Rakefile +12 -0
  12. data/bin/console +15 -0
  13. data/bin/rake +29 -0
  14. data/bin/rubocop +29 -0
  15. data/bin/setup +8 -0
  16. data/bin/srb +29 -0
  17. data/lib/minitest/distributed.rb +36 -0
  18. data/lib/minitest/distributed/configuration.rb +53 -0
  19. data/lib/minitest/distributed/coordinators/coordinator_interface.rb +29 -0
  20. data/lib/minitest/distributed/coordinators/memory_coordinator.rb +67 -0
  21. data/lib/minitest/distributed/coordinators/redis_coordinator.rb +387 -0
  22. data/lib/minitest/distributed/enqueued_runnable.rb +88 -0
  23. data/lib/minitest/distributed/filters/exclude_filter.rb +35 -0
  24. data/lib/minitest/distributed/filters/filter_interface.rb +25 -0
  25. data/lib/minitest/distributed/filters/include_filter.rb +35 -0
  26. data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb +76 -0
  27. data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb +48 -0
  28. data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb +61 -0
  29. data/lib/minitest/distributed/result_aggregate.rb +67 -0
  30. data/lib/minitest/distributed/result_type.rb +28 -0
  31. data/lib/minitest/distributed/test_runner.rb +37 -0
  32. data/lib/minitest/distributed/test_selector.rb +54 -0
  33. data/lib/minitest/distributed/version.rb +8 -0
  34. data/lib/minitest/distributed_plugin.rb +51 -0
  35. data/minitest-distributed.gemspec +50 -0
  36. data/sorbet/config +2 -0
  37. data/sorbet/rbi/minitest.rbi +238 -0
  38. data/sorbet/rbi/rbconfig.rbi +6 -0
  39. data/sorbet/rbi/redis.rbi +70 -0
  40. data/sorbet/rbi/winsize.rbi +7 -0
  41. metadata +142 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: c7cdf0e712969711ce22cfa2f7344b112177eff46d2e632a447b71830a248761
4
+ data.tar.gz: 5c4e0ebdaf02176ad90d4ae6f907f5fc619c6335f4df64f4667eaaf54e7e657e
5
+ SHA512:
6
+ metadata.gz: 1858ec3fa36a8f31e2210999d4b5db0d361d6e01169946ecd2f9aec2af19028502d908301dd19e9657be8171001479a0914650a195013c6672f54305f639df87
7
+ data.tar.gz: e720806c0fefa3958adb4c8170226913eb5c587819eef3666dde63018aacd882355d816384291029c260fc39681bf37ad4140f72cdabb2028303e259d8229c4d
@@ -0,0 +1,48 @@
1
+ # This workflow uses actions that are not certified by GitHub.
2
+ # They are provided by a third-party and are governed by
3
+ # separate terms of service, privacy policy, and support
4
+ # documentation.
5
+ # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
6
+ # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
7
+
8
+ name: Ruby
9
+
10
+ on: push
11
+
12
+ jobs:
13
+ test:
14
+ runs-on: ubuntu-latest
15
+ container: ruby:2.6
16
+
17
+ services:
18
+ redis:
19
+ image: redis
20
+
21
+ steps:
22
+ - uses: actions/checkout@v2
23
+ - name: Install dependencies
24
+ run: gem install bundler && bundle install
25
+ - name: Run tests
26
+ run: bin/rake test
27
+ env:
28
+ REDIS_URL: redis://redis:6379
29
+
30
+ typecheck:
31
+ runs-on: ubuntu-latest
32
+ container: ruby:2.6
33
+ steps:
34
+ - uses: actions/checkout@v2
35
+ - name: Install dependencies
36
+ run: gem install bundler && bundle install
37
+ - name: Typecheck Ruby code
38
+ run: bin/srb tc
39
+
40
+ lint:
41
+ runs-on: ubuntu-latest
42
+ container: ruby:2.6
43
+ steps:
44
+ - uses: actions/checkout@v2
45
+ - name: Install dependencies
46
+ run: gem install bundler && bundle install
47
+ - name: Lint Ruby code
48
+ run: bin/rubocop
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
@@ -0,0 +1,63 @@
1
+ inherit_gem:
2
+ rubocop-shopify: rubocop.yml
3
+
4
+ require:
5
+ - rubocop-sorbet
6
+
7
+ AllCops:
8
+ TargetRubyVersion: 2.6
9
+ UseCache: true
10
+ CacheRootDirectory: tmp/rubocop
11
+ Exclude:
12
+ - minitest-distributed.gemspec
13
+
14
+ Style/MethodCallWithArgsParentheses:
15
+ IgnoredMethods:
16
+ - require
17
+ - require_relative
18
+ - raise
19
+ - assert
20
+ - refute
21
+ - assert_equal
22
+ - refute_equal
23
+ - assert_nil
24
+ - refute_nil
25
+ - assert_predicate
26
+ - refute_predicate
27
+ - assert_empty
28
+ - refute_empty
29
+ - assert_includes
30
+ - refute_includes
31
+ - flunk
32
+
33
+ ##### Sorbet cops
34
+
35
+ Sorbet:
36
+ Enabled: true
37
+ Exclude:
38
+ - bin/*
39
+ - test/fixtures/*
40
+ - Gemfile
41
+ - Rakefile
42
+ - minitest-distributed.gemspec
43
+
44
+ Sorbet/ValidSigil:
45
+ Enabled: true
46
+
47
+ Sorbet/TrueSigil:
48
+ Enabled: true
49
+
50
+ Sorbet/EnforceSigilOrder:
51
+ Enabled: true
52
+
53
+ Sorbet/SignatureBuildOrder:
54
+ Enabled: true
55
+
56
+ Sorbet/KeywordArgumentOrdering:
57
+ Enabled: true
58
+
59
+ Sorbet/ConstantsFromStrings:
60
+ Enabled: true
61
+
62
+ Sorbet/ForbidIncludeConstLiteral:
63
+ Enabled: true
@@ -0,0 +1,6 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.6.5
6
+ before_install: gem install bundler -v 2.1.4
@@ -0,0 +1,74 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to making participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, gender identity and expression, level of experience,
9
+ nationality, personal appearance, race, religion, or sexual identity and
10
+ orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies both within project spaces and in public spaces
49
+ when an individual is representing the project or its community. Examples of
50
+ representing a project or community include using an official project e-mail
51
+ address, posting via an official social media account, or acting as an appointed
52
+ representative at an online or offline event. Representation of a project may be
53
+ further defined and clarified by project maintainers.
54
+
55
+ ## Enforcement
56
+
57
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
58
+ reported by contacting the project team at willem@vanbergen.org. All
59
+ complaints will be reviewed and investigated and will result in a response that
60
+ is deemed necessary and appropriate to the circumstances. The project team is
61
+ obligated to maintain confidentiality with regard to the reporter of an incident.
62
+ Further details of specific enforcement policies may be posted separately.
63
+
64
+ Project maintainers who do not follow or enforce the Code of Conduct in good
65
+ faith may face temporary or permanent repercussions as determined by other
66
+ members of the project's leadership.
67
+
68
+ ## Attribution
69
+
70
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71
+ available at [https://contributor-covenant.org/version/1/4][version]
72
+
73
+ [homepage]: https://contributor-covenant.org
74
+ [version]: https://contributor-covenant.org/version/1/4/
data/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+ source "https://rubygems.org"
3
+
4
+ # Specify your gem's dependencies in minitest-stateful.gemspec
5
+ gemspec
6
+
7
+ gem "rake", "~> 12.0"
8
+ gem "minitest", "~> 5.0"
9
+ gem "sorbet"
10
+ gem "rubocop"
11
+ gem "rubocop-shopify"
12
+ gem "rubocop-sorbet"
@@ -0,0 +1,53 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ minitest-distributed (0.1.0)
5
+ minitest (~> 5.12)
6
+ redis (~> 4.1)
7
+ sorbet-runtime
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ ast (2.4.0)
13
+ jaro_winkler (1.5.4)
14
+ minitest (5.14.0)
15
+ parallel (1.19.1)
16
+ parser (2.7.1.2)
17
+ ast (~> 2.4.0)
18
+ rainbow (3.0.0)
19
+ rake (12.3.3)
20
+ redis (4.2.1)
21
+ rexml (3.2.4)
22
+ rubocop (0.82.0)
23
+ jaro_winkler (~> 1.5.1)
24
+ parallel (~> 1.10)
25
+ parser (>= 2.7.0.1)
26
+ rainbow (>= 2.2.2, < 4.0)
27
+ rexml
28
+ ruby-progressbar (~> 1.7)
29
+ unicode-display_width (>= 1.4.0, < 2.0)
30
+ rubocop-shopify (1.0.2)
31
+ rubocop (~> 0.82.0)
32
+ rubocop-sorbet (0.3.7)
33
+ ruby-progressbar (1.10.1)
34
+ sorbet (0.5.5605)
35
+ sorbet-static (= 0.5.5605)
36
+ sorbet-runtime (0.5.5765)
37
+ sorbet-static (0.5.5605-universal-darwin-14)
38
+ unicode-display_width (1.7.0)
39
+
40
+ PLATFORMS
41
+ ruby
42
+
43
+ DEPENDENCIES
44
+ minitest (~> 5.0)
45
+ minitest-distributed!
46
+ rake (~> 12.0)
47
+ rubocop
48
+ rubocop-shopify
49
+ rubocop-sorbet
50
+ sorbet
51
+
52
+ BUNDLED WITH
53
+ 2.1.4
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Willem van Bergen, Shopify Inc.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,115 @@
1
+ # minitest-distributed
2
+
3
+ `minitest-distributed` is a plugin for [minitest](https://github.com/seattlerb/minitest)
4
+ for executing tests on a distributed set of unreliable workers.
5
+
6
+ When a test suite grows large enough, it inevitable gets too slow to run on a
7
+ single machine to give timely feedback to developers. This plugins combats
8
+ this issue by distributing the full test suite to a set of workers. Every
9
+ worker is a consuming from a single queue, so the tests get evenly distributed
10
+ and all workers will finish around the same time. Redis is used as
11
+ coordinator, but when using this plugin without having access to Redis, it
12
+ will use an in-memory coordinator.
13
+
14
+ Using multiple (virtual) machines for a test run is an (additional) source of
15
+ flakiness. To combat flakiness, minitest-distributed implements resiliency
16
+ patterns, like re-running a test on a different worker on failure, and a
17
+ circuit breaker for misbehaving workers.
18
+
19
+ ## Usage
20
+
21
+ Add `minitest-distributed` to your `Gemfile`, and run `bundle install`. The
22
+ plugin will be loaded by minitest automatically. The plugin exposes some
23
+ command line arguments that you can use to influence its behavior. They can
24
+ also be set using environment variables.
25
+
26
+ ### Distributed invocation
27
+
28
+ To actually run tests with multiple workers, you have to point every worker to
29
+ a Redis coordinator, and use the same run identifier.
30
+
31
+ ``` sh
32
+ ruby -Itest --coordinator=redis://localhost/1 --run-id=<BUILD_NUMBER> test/my_test.rb
33
+ ```
34
+
35
+ We recommend using the build number or identifier form your CI system as run
36
+ identifier, but it will work with any value that is shared between all
37
+ workers. You can also set these values by setting the `MINITEST_COORDINATOR`
38
+ and `MINITEST_RUN_ID` environment variables, respectively.
39
+
40
+ If you are using a Rails project, the Rails test runner (`bin/rails test`)
41
+ will also support these command line arguments and environment variables.
42
+
43
+ If you are using a `Rake::TestTask` to invoke your test suite, you can set
44
+ these command line arguments using `options`:
45
+
46
+ ``` ruby
47
+ Rake::TestTask.new(:test) do |t|
48
+ t.libs << "test" << "lib"
49
+ t.warning = false
50
+ t.options = "--coordinator=redis://localhost/1 --run-id=<BUILD_NUMBER>"
51
+ t.test_files = FileList["test/**/*_test.rb"]
52
+ end
53
+ ```
54
+
55
+ ### Worker retries
56
+
57
+ Many CI systems offer the options to retry jobs that fail. When jobs are
58
+ retried that were previously part of a worker cluster, all the retried jobs
59
+ together will form a new cluster, and will only run the tests that failed
60
+ during the previous run attempt. This is to make it faster to re-run tests
61
+ that failed due to flakiness, or confirm that it was not flakiness that caused
62
+ them to fail.
63
+
64
+ ### Other optional command line arguments
65
+
66
+ - `--test-timeout=SECONDS` or `ENV[MINITEST_TEST_TIMEOUT]` (default: 30s): the
67
+ maximum amount a test is allowed to run before it times out. In a distributed
68
+ system, it's impossible to differentiate between a worker being slow and a
69
+ worker being broken. When the timeout passes, the other workers will assume
70
+ that the worker running the test has crashed, and will attempt to claim this
71
+ test. This value should be comfortably higher than your slowest test.
72
+ - `--max-attempts=NUMBER` or `ENV[MINITEST_MAX_ATTEMPTS]` (default: 3). The
73
+ maximum number of times a test is attempted to be run, before considering
74
+ it failed. Higher values will prevent more flakiness, but will make the full
75
+ test run slower.
76
+ - `--test-batch-size=NUMBER` or `ENV[MINITEST_TEST_BATCH_SIZE]` (default: 10).
77
+ The amount of tests to process per batch. Lower numbers will make the
78
+ distribution of tests more granular and even, but increase the load on the
79
+ coordinator.
80
+ - `--worker-id=IDENTIFIER` or `ENV[MINITEST_WORKER_ID]`: The ID of the worker,
81
+ which should be unique to the cluster. We will default to a UUID if this is
82
+ not set, which generally is fine.
83
+
84
+ ## Limitations
85
+
86
+ **Parallel tests not supported:** Minitest comes bundled with a parallel test
87
+ executor, which will run tests that are specifically tagged as such in
88
+ parallel in the same process. `minitest-distributed` is designed to run tests
89
+ in parallel using separate processes, generally on different VMs. For this
90
+ reason, tests marked as `parallel` will not be treated any differently than
91
+ other tests.
92
+
93
+ ## Development
94
+
95
+ After checking out the repo, run `bin/setup` to install dependencies. Then,
96
+ run `rake test` to run the tests. You can also run `bin/console` for an
97
+ interactive prompt that will allow you to experiment.
98
+
99
+ To install this gem onto your local machine, run `bundle exec rake install`.
100
+ To release a new version, update the version number in `version.rb`, and then
101
+ run `bundle exec rake release`, which will create a git tag for the version,
102
+ push git commits and tags, and push the `.gem` file to
103
+ [rubygems.org](https://rubygems.org).
104
+
105
+ ## Contributing
106
+
107
+ Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/minitest-distributed. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).
108
+
109
+ ## License
110
+
111
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
112
+
113
+ ## Code of Conduct
114
+
115
+ Everyone interacting in the Minitest::Stateful project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+ require "bundler/gem_tasks"
3
+ require "rake/testtask"
4
+
5
+ Rake::TestTask.new(:test) do |t|
6
+ t.libs << "test" << "lib"
7
+ t.warning = false
8
+ t.test_files = FileList["test/**/*_test.rb"] -
9
+ FileList["test/fixtures/**/*_test.rb"] - FileList["test/lib/**/*_test.rb"]
10
+ end
11
+
12
+ task(default: :test)
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "bundler/setup"
5
+ require "minitest/distributed"
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ # (If you use this, don't forget to add pry to your Gemfile!)
11
+ # require "pry"
12
+ # Pry.start
13
+
14
+ require "irb"
15
+ IRB.start(__FILE__)