minitest-distributed 0.1.2 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfef30635f4dd487b913e94e04a1098f27d27278af1f1adbb794a1f91f54e880
-  data.tar.gz: cff09c3e45440e77a7f39dabcdc6faf3a99336cc05362ea1740c810c91bb227b
+  metadata.gz: 6a8b0499822e88334b7ce7d06210c43d3054485d4dbbc3170f97e485c5d78c3d
+  data.tar.gz: 2e4c720af2bf722152ace13867b0cf66c58aafbf712ac1a5d4b9e308f0c2fd8f
 SHA512:
-  metadata.gz: b9a05e26c4b0a432d3b7c2d8aec342d141b0fc5b4a1b7a355b6e5e14609a85e834c60fde5d6bf32a85529d9678c0ce19e2a8e9feb1d6da49c761b77289c340b0
-  data.tar.gz: 430c1c0da35af7e4db05580676de9ae4d911ecb0673d99535b07c00a72f600dd71e3e279a127e254bba2be13352579d1867ff376edaf1b91fd241b5b6da0b408
+  metadata.gz: 8c974a610c8770a9a0ff8c944780182a178fbe1d22737a53a474b9f8f2abcd90765aac1f6a0ee9134cddb1e19dca235bb124b7950585337579b7dba5d4ebf956
+  data.tar.gz: 8337bf3b5c849d386ccbdb13e4cba6d725a274ead9893c60f3d60f9599e5ba7cde0e12ceea035219483ebd9aee0fc06ee9cc9e6a4747c08663f133dab30fd4d0

data/.rubocop.yml

@@ -11,6 +11,10 @@ AllCops:
   Exclude:
     - minitest-distributed.gemspec
 
+# This cop is broken when using assignments
+Layout/RescueEnsureAlignment:
+  Enabled: false
+
 ##### Sorbet cops
 
 Sorbet:

data/Gemfile

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in minitest-stateful.gemspec
+# Specify your gem's dependencies in minitest-distributed.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake"
 gem "minitest", "~> 5.0"
 gem "sorbet"
 gem "rubocop"

data/README.md

@@ -63,8 +63,8 @@ them to fail.
 
 ### Other optional command line arguments
 
-- `--test-timeout=SECONDS` or `ENV[MINITEST_TEST_TIMEOUT]` (default: 30s): the
-  maximum amount a test is allowed to run before it times out. In a distributed
+- `--test-timeout=SECONDS` or `ENV[MINITEST_TEST_TIMEOUT_SECONDS]` (default: 30s):
+  the maximum amount a test is allowed to run before it times out. In a distributed
   system, it's impossible to differentiate between a worker being slow and a
   worker being broken. When the timeout passes, the other workers will assume
   that the worker running the test has crashed, and will attempt to claim this
@@ -80,6 +80,12 @@ them to fail.
 - `--worker-id=IDENTIFIER` or `ENV[MINITEST_WORKER_ID]`: The ID of the worker,
   which should be unique to the cluster. We will default to a UUID if this is
   not set, which generally is fine.
+- `--exclude-file=PATH_TO_FILE`: Specify a file of tests to be excluded 
+  from running. The file should include test identifiers seperated by 
+  newlines.
+- `--include-file=PATH_TO_FILE`: Specify a file of tests to be included in
+  the test run. The file should include test identifiers seperated by 
+  newlines.
 
 ## Limitations
 
@@ -92,24 +98,40 @@ other tests.
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then,
-run `rake test` to run the tests. You can also run `bin/console` for an
-interactive prompt that will allow you to experiment.
+To bootstrap a local development environment:
 
-To install this gem onto your local machine, run `bundle exec rake install`.
-To release a new version, update the version number in `version.rb`, and then
-run `bundle exec rake release`, which will create a git tag for the version,
-push git commits and tags, and push the `.gem` file to
-[rubygems.org](https://rubygems.org).
+- Run `bin/setup` to install dependencies.
+- Start a Redis server by running `redis-server`, assuming you have Redis
+  installed locally and the binary is on your `PATH`. Alternatively, you can
+  set the `REDIS_URL` environment variable to point to a Redis instance running
+  elsewhere.
+- Now, run `bin/rake test` to run the tests, and verify everything is working.
+- You can also run `bin/console` for an interactive prompt that will allow you
+  to experiment.
+
+### Releasing a new version
+
+- To install this gem onto your local machine, run `bin/rake install`.
+- Only people at Shopify can release a new version to
+  [rubygems.org](https://rubygems.org). To do so, update the `VERSION` constant
+  in `version.rb`, and merge to master. Shipit will take care of building the
+  `.gem` bundle, and pushing it to rubygems.org.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/minitest-distributed. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/Shopify/minitest-distributed. This project is intended to
+be a safe, welcoming space for collaboration, and contributors are expected to
+adhere to the [code of
+conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).
 
 ## Code of Conduct
 
-Everyone interacting in the Minitest::Stateful project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the `minitest-distributed` project's codebases, issue
+trackers, chat rooms and mailing lists is expected to follow the [code of
+conduct](https://github.com/Shopify/minitest-distributed/blob/master/CODE_OF_CONDUCT.md).

data/bin/setup

@@ -4,5 +4,3 @@ IFS=$'\n\t'
 set -vx
 
 bundle install
-
-# Do any other automated setup that you need to do here

data/lib/minitest/distributed/configuration.rb

@@ -8,8 +8,8 @@ module Minitest
   module Distributed
     class Configuration < T::Struct
       DEFAULT_BATCH_SIZE = 10
-      DEFAULT_MAX_ATTEMPTS = 3
-      DEFAULT_TEST_TIMEOUT = 30.0 # seconds
+      DEFAULT_MAX_ATTEMPTS = 1
+      DEFAULT_TEST_TIMEOUT_SECONDS = 30.0 # seconds
 
       class << self
         extend T::Sig
@@ -20,21 +20,83 @@ module Minitest
             coordinator_uri: URI(env['MINITEST_COORDINATOR'] || 'memory:'),
             run_id: env['MINITEST_RUN_ID'] || SecureRandom.uuid,
             worker_id: env['MINITEST_WORKER_ID'] || SecureRandom.uuid,
-            test_timeout: Float(env['MINITEST_TEST_TIMEOUT'] || DEFAULT_TEST_TIMEOUT),
+            test_timeout_seconds: Float(env['MINITEST_TEST_TIMEOUT_SECONDS'] || DEFAULT_TEST_TIMEOUT_SECONDS),
             test_batch_size: Integer(env['MINITEST_TEST_BATCH_SIZE'] || DEFAULT_BATCH_SIZE),
             max_attempts: Integer(env['MINITEST_MAX_ATTEMPTS'] || DEFAULT_MAX_ATTEMPTS),
+            max_failures: (max_failures_env = env['MINITEST_MAX_FAILURES']) ? Integer(max_failures_env) : nil,
           )
         end
+
+        sig { params(opts: OptionParser, options: T::Hash[Symbol, T.untyped]).returns(T.attached_class) }
+        def from_command_line_options(opts, options)
+          configuration = from_env
+          configuration.progress = options[:io].tty?
+
+          opts.on('--coordinator=URI', "The URI pointing to the coordinator") do |uri|
+            configuration.coordinator_uri = URI.parse(uri)
+          end
+
+          opts.on('--test-timeout=TIMEOUT', "The maximum run time for a single test in seconds") do |timeout|
+            configuration.test_timeout_seconds = Float(timeout)
+          end
+
+          opts.on('--max-attempts=ATTEMPTS', "The maximum number of attempts to run a test") do |attempts|
+            configuration.max_attempts = Integer(attempts)
+          end
+
+          opts.on('--test-batch-size=NUMBER', "The number of tests to process per batch") do |batch_size|
+            configuration.test_batch_size = Integer(batch_size)
+          end
+
+          opts.on('--max-failures=FAILURES', "The maximum allowed failure before aborting a run") do |failures|
+            configuration.max_failures = Integer(failures)
+          end
+
+          opts.on('--run-id=ID', "The ID for this run shared between coordinated workers") do |id|
+            configuration.run_id = id
+          end
+
+          opts.on('--worker-id=ID', "The unique ID for this worker") do |id|
+            configuration.worker_id = id
+          end
+
+          opts.on(
+            '--[no-]retry-failures', "Retry failed and errored tests from a previous run attempt " \
+            "with the same run ID (default: enabled)"
+          ) do |enabled|
+            configuration.retry_failures = enabled
+          end
+
+          opts.on('--[no-]progress', "Show progress during the test run") do |enabled|
+            configuration.progress = enabled
+          end
+
+          opts.on('--exclude-file=FILE_PATH', "Specify a file of tests to be excluded from running") do |file_path|
+            configuration.exclude_file = file_path
+          end
+
+          opts.on('--include-file=FILE_PATH', "Specify a file of tests to be included in the test run") do |file_path|
+            configuration.include_file = file_path
+          end
+
+          configuration
+        end
       end
 
       extend T::Sig
 
+      # standard minitest options don't need to be specified
       prop :coordinator_uri, URI::Generic, default: URI('memory:')
       prop :run_id, String, factory: -> { SecureRandom.uuid }
       prop :worker_id, String, factory: -> { SecureRandom.uuid }
-      prop :test_timeout, Float, default: DEFAULT_TEST_TIMEOUT
+      prop :test_timeout_seconds, Float, default: DEFAULT_TEST_TIMEOUT_SECONDS
       prop :test_batch_size, Integer, default: DEFAULT_BATCH_SIZE
       prop :max_attempts, Integer, default: DEFAULT_MAX_ATTEMPTS
+      prop :max_failures, T.nilable(Integer)
+      prop :retry_failures, T::Boolean, default: true
+      prop :progress, T::Boolean, default: false
+      prop :exclude_file, T.nilable(String)
+      prop :include_file, T.nilable(String)
 
       sig { returns(Coordinators::CoordinatorInterface) }
       def coordinator

data/lib/minitest/distributed/coordinators/coordinator_interface.rb

@@ -18,6 +18,9 @@ module Minitest
         sig { abstract.returns(ResultAggregate) }
         def combined_results; end
 
+        sig { abstract.returns(T::Boolean) }
+        def aborted?; end
+
         sig { abstract.params(test_selector: TestSelector).void }
         def produce(test_selector:); end
 

data/lib/minitest/distributed/coordinators/memory_coordinator.rb

@@ -25,7 +25,8 @@ module Minitest
 
           @leader = T.let(Mutex.new, Mutex)
           @queue = T.let(Queue.new, Queue)
-          @local_results = T.let(ResultAggregate.new, ResultAggregate)
+          @local_results = T.let(ResultAggregate.new(max_failures: configuration.max_failures), ResultAggregate)
+          @aborted = T.let(false, T::Boolean)
         end
 
         sig { override.params(reporter: Minitest::CompositeReporter, options: T::Hash[Symbol, T.untyped]).void }
@@ -33,6 +34,11 @@ module Minitest
           # No need for any additional reporters
         end
 
+        sig { override.returns(T::Boolean) }
+        def aborted?
+          @aborted
+        end
+
         sig { override.params(test_selector: TestSelector).void }
         def produce(test_selector:)
           if @leader.try_lock
@@ -41,24 +47,39 @@ module Minitest
             if tests.empty?
               queue.close
             else
-              tests.each { |test| queue << test }
+              tests.each do |runnable|
+                queue << EnqueuedRunnable.new(
+                  class_name: T.must(runnable.class.name),
+                  method_name: runnable.name,
+                  test_timeout_seconds: configuration.test_timeout_seconds,
+                  max_attempts: configuration.max_attempts,
+                )
+              end
             end
           end
         end
 
         sig { override.params(reporter: AbstractReporter).void }
         def consume(reporter:)
-          until queue.empty? && queue.closed?
-            enqueued_runnable = queue.pop
+          until queue.closed?
+            enqueued_runnable = T.let(queue.pop, EnqueuedRunnable)
+
             reporter.prerecord(enqueued_runnable.runnable_class, enqueued_runnable.method_name)
-            result = enqueued_runnable.run
 
-            local_results.update_with_result(result)
-            local_results.acks += 1
+            initial_result = enqueued_runnable.run
+            enqueued_result = enqueued_runnable.commit_result(initial_result) do |result_to_commit|
+              if ResultType.of(result_to_commit) == ResultType::Requeued
+                queue << enqueued_runnable.next_attempt
+              end
+              EnqueuedRunnable::Result::Commit.success
+            end
 
-            reporter.record(result)
+            reporter.record(enqueued_result.committed_result)
+            local_results.update_with_result(enqueued_result)
 
-            queue.close if local_results.completed?
+            # We abort a run if we reach the maximum number of failures
+            queue.close if combined_results.abort?
+            queue.close if combined_results.complete?
           end
         end
       end