minitest-distributed 0.2.10 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a59856c10fd7a252d95609f66242a4c9706f91d6e7fb57a226914a7e816e04ee
-  data.tar.gz: 024e72b9de525b048dbdc0a6f97b7721a24fd39803030dec85cf080832dfcfd3
+  metadata.gz: 722ec2cc00da4f9c5f5efc0591e39c2becc2338ce7fe171c60339d284116242b
+  data.tar.gz: aa1d3c52afa3b16c649a0b12d27e83ccb19ae11dd80fb68d3e57a669e0a081e5
 SHA512:
-  metadata.gz: 401e8fcb14acf32f0e3904038e30d15bbc801e1cf7672c93b16c09d506829c46dc58fff771766ddd508e46d8740748a5688a8c0125ce19bafa8fe2dd16a81909
-  data.tar.gz: d0082e9f6d731b73651e26c6a6ea943ff782b93cca2c11de19308b1e97a415f06450770a858ea6e176153df5183710a6dd2f2bc327474dde930b1d8685f40c08
+  metadata.gz: c2cfc291aeb202bb680b04a6e579272fd907efe47ef1d5e5ea2b66ca5ffe23967ded9ea38458e485e1e9b42609d84a2058fd2df3250217cf76a60272f3b3255c
+  data.tar.gz: cb0934c54f7c3bdcc76abe8c0593b08e2e7ddfacd9cf2c4e7ce3a71e586b2e75170be91ebe3ac8b520fb7c8c7e4e961ae4a9a218c582bf1301dc916f41427847

data/.rubocop.yml

@@ -9,7 +9,6 @@ require:
   - rubocop-minitest
 
 AllCops:
-  TargetRubyVersion: 2.6
   UseCache: true
   CacheRootDirectory: tmp/rubocop
   Exclude:

data/.ruby-version

@@ -0,0 +1 @@
+3.2.2

data/README.md

@@ -23,12 +23,6 @@ flakiness. To combat flakiness, minitest-distributed implements resiliency
 patterns, like re-running a test on a different worker on failure, and a
 circuit breaker for misbehaving workers.
 
-|               |                                                                                                                                      |
-|----------------|--------------------------------------------------------------------------------------------------------------------------------------|
-| Current status | Ongoing                                                                                                                              |
-| Owner          | [@Shopify/test-infra](https://github.com/orgs/Shopify/teams/test-infra)                                                              |
-| Help           | [#team-test-infra](https://shopify.slack.com/archives/team-test-infra)                                                               |
-
 ## Commands
 **Distributed invocation**
 

data/dev.yml

@@ -3,5 +3,6 @@ name: minitest-distributed
 type: ruby
 
 up:
-  - ruby: 3.2.0
+  - ruby
   - bundler
+  - redis

data/lib/minitest/distributed/configuration.rb

@@ -105,15 +105,17 @@ module Minitest
 
       sig { returns(Coordinators::CoordinatorInterface) }
       def coordinator
-        @coordinator = T.let(@coordinator, T.nilable(Coordinators::CoordinatorInterface))
-        @coordinator ||= case coordinator_uri.scheme
-        when "redis"
-          Coordinators::RedisCoordinator.new(configuration: self)
-        when "memory"
-          Coordinators::MemoryCoordinator.new(configuration: self)
-        else
-          raise NotImplementedError, "Unknown coordinator implementation: #{coordinator_uri.scheme}"
-        end
+        @coordinator ||= T.let(
+          case coordinator_uri.scheme
+          when "redis"
+            Coordinators::RedisCoordinator.new(configuration: self)
+          when "memory"
+            Coordinators::MemoryCoordinator.new(configuration: self)
+          else
+            raise NotImplementedError, "Unknown coordinator implementation: #{coordinator_uri.scheme}"
+          end,
+          T.nilable(Coordinators::CoordinatorInterface),
+        )
       end
     end
   end

data/lib/minitest/distributed/coordinators/redis_coordinator.rb

@@ -3,6 +3,7 @@
 
 require "redis"
 require "set"
+require "logger"
 
 module Minitest
   module Distributed
@@ -268,13 +269,30 @@ module Minitest
 
         sig { returns(Redis) }
         def redis
-          @redis ||= Redis.new(url: configuration.coordinator_uri.to_s)
+          @redis ||= Redis.new(
+            url: configuration.coordinator_uri.to_s,
+            middlewares: custom_middlewares,
+            custom: custom_config,
+            timeout: 2,
+          )
+        end
+
+        sig { returns(T.nilable(T::Array[Module])) }
+        def custom_middlewares
+          return unless ENV.key?("MINITEST_DISTRIBUTED_REDIS_LOG")
+
+          require_relative "redis_instrumentation_middleware"
+          [RedisInstrumentationMiddleware]
+        end
+
+        sig { returns(T.nilable(T::Hash[Symbol, File])) }
+        def custom_config
+          { log_file: logger }.compact
         end
 
         sig { returns(String) }
         def register_consumergroup_script
-          @register_consumergroup_script = T.let(@register_consumergroup_script, T.nilable(String))
-          @register_consumergroup_script ||= redis.script(:load, <<~LUA)
+          @register_consumergroup_script ||= T.let(redis.script(:load, <<~LUA), T.nilable(String))
             -- Try to create the consumergroup. This will raise an error if the
             -- consumergroup has already been registered by somebody else, which
             -- means another worker will be acting as leader.
@@ -518,6 +536,13 @@ module Minitest
           adjust_combined_results(batch_result_aggregate)
         end
 
+        sig { returns(T.nilable(Logger)) }
+        def logger
+          return unless (log_path = ENV["MINITEST_DISTRIBUTED_REDIS_LOG"])
+
+          @logger ||= T.let(Logger.new(log_path), T.nilable(Logger))
+        end
+
         INITIAL_BACKOFF = 10 # milliseconds
         private_constant :INITIAL_BACKOFF
       end

data/lib/minitest/distributed/coordinators/redis_instrumentation_middleware.rb

@@ -0,0 +1,39 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "redis"
+
+module Minitest
+  module Distributed
+    module Coordinators
+      # Redis middleware that logs all Redis commands to a file for debugging.
+      module RedisInstrumentationMiddleware
+        extend T::Sig
+
+        sig { params(command: T::Array[T.untyped], redis_config: T.untyped).returns(T.untyped) }
+        def call(command, redis_config)
+          log_file = redis_config.custom[:log_file]
+          log_file.info("EXEC: #{command.inspect}")
+          result = super
+          log_file.info("RESULT: #{result.inspect}")
+          result
+        rescue => e
+          log_file.info("ERROR: #{e.class}")
+          Kernel.raise
+        end
+
+        sig { params(commands: T::Array[T.untyped], redis_config: T.untyped).returns(T.untyped) }
+        def call_pipelined(commands, redis_config)
+          log_file = redis_config.custom[:log_file]
+          log_file.info("EXEC PIPELINED: #{commands.inspect}")
+          result = super
+          log_file.info("RESULT PIPELINED: #{result.inspect}")
+          result
+        rescue => e
+          log_file.info("ERROR PIPELINED: #{e.class}")
+          Kernel.raise
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/enqueued_runnable.rb

@@ -61,8 +61,7 @@ module Minitest
 
           sig { returns(T::Boolean) }
           def success?
-            @success = T.let(@success, T.nilable(T::Boolean))
-            @success ||= @block.call
+            @success ||= T.let(@block.call, T.nilable(T::Boolean))
           end
 
           sig { returns(T::Boolean) }
@@ -72,14 +71,12 @@ module Minitest
 
           sig { returns(Commit) }
           def self.success
-            @success = T.let(@success, T.nilable(Commit))
-            @success ||= new { true }
+            @success ||= T.let(new { true }, T.nilable(Commit))
           end
 
           sig { returns(Commit) }
           def self.failure
-            @failure = T.let(@failure, T.nilable(Commit))
-            @failure ||= new { false }
+            @failure ||= T.let(new { false }, T.nilable(Commit))
           end
         end
 
@@ -106,13 +103,18 @@ module Minitest
 
         sig { returns(Minitest::Result) }
         def committed_result
-          @committed_result = T.let(@committed_result, T.nilable(Minitest::Result))
-          @committed_result ||= if final? && commit.failure?
-            # If a runnable result is final, but the acked failed, we will discard the result.
-            Minitest::Discard.wrap(initial_result, test_timeout_seconds: enqueued_runnable.test_timeout_seconds)
-          else
-            initial_result
-          end
+          @committed_result ||= T.let(
+            if final? && commit.failure?
+              # If a runnable result is final, but the acked failed, we will discard the result.
+              Minitest::Discard.wrap(
+                initial_result,
+                test_timeout_seconds: enqueued_runnable.test_timeout_seconds,
+              )
+            else
+              initial_result
+            end,
+            T.nilable(Minitest::Result),
+          )
         end
       end
 

data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb

@@ -67,14 +67,12 @@ module Minitest
 
         sig { returns(ResultAggregate) }
         def local_results
-          @local_results = T.let(@local_results, T.nilable(ResultAggregate))
-          @local_results ||= configuration.coordinator.local_results
+          @local_results ||= T.let(configuration.coordinator.local_results, T.nilable(ResultAggregate))
         end
 
         sig { returns(ResultAggregate) }
         def combined_results
-          @combined_results = T.let(@combined_results, T.nilable(ResultAggregate))
-          @combined_results ||= configuration.coordinator.combined_results
+          @combined_results ||= T.let(configuration.coordinator.combined_results, T.nilable(ResultAggregate))
         end
 
         sig { returns(Configuration) }

data/lib/minitest/distributed/version.rb

@@ -3,6 +3,6 @@
 
 module Minitest
   module Distributed
-    VERSION = "0.2.10"
+    VERSION = "0.2.12"
   end
 end

data/lib/minitest/distributed_plugin.rb

@@ -1,7 +1,7 @@
 # typed: true
 # frozen_string_literal: true
 
-require_relative "./distributed"
+require_relative "distributed"
 
 module Minitest
   class << self

data/minitest-distributed.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
 
   spec.homepage      = "https://github.com/Shopify/minitest-distributed"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = ">= 2.7.0"
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|.shopify-build|.github)/}) }
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -47,4 +47,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency('minitest', '~> 5.12')
   spec.add_dependency('redis', '>= 5.0.6', '< 6')
   spec.add_dependency('sorbet-runtime')
+  spec.add_dependency('rexml')
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: minitest-distributed
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.12
 platform: ruby
 authors:
 - Willem van Bergen
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -58,6 +57,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |
   minitest-distributed is a plugin for minitest for executing tests on a
   distributed set of unreliable workers.
@@ -80,12 +93,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/ISSUE_TEMPLATE/bug_report.md"
-- ".github/dependabot.yml"
-- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- ".ruby-version"
 - CODEOWNERS
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -104,6 +115,7 @@ files:
 - lib/minitest/distributed/coordinators/coordinator_interface.rb
 - lib/minitest/distributed/coordinators/memory_coordinator.rb
 - lib/minitest/distributed/coordinators/redis_coordinator.rb
+- lib/minitest/distributed/coordinators/redis_instrumentation_middleware.rb
 - lib/minitest/distributed/enqueued_runnable.rb
 - lib/minitest/distributed/filters/exclude_file_filter.rb
 - lib/minitest/distributed/filters/exclude_filter.rb
@@ -157,7 +169,6 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/Shopify/minitest-distributed
   source_code_uri: https://github.com/Shopify/minitest-distributed
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -165,15 +176,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.18
-signing_key: 
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Distributed test executor plugin for Minitest
 test_files: []

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,43 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: "[BUG]"
-labels: bug
-assignees: ''
-
----
-
-**Describe the issue**
-A clear and concise description of the issue you're facing.
-
-**To Reproduce**
-Steps to reproduce the behaviour:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behaviour**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**What team are you on?**
- Please provide the name of your team.
-
-**Helpful links (Logs, Link to Build, Repo, PR etc.)** 
-Please provide any helpful links for investigating the issue.
-
-**Is there a Slack thread related to this bug?** 
-After submitting this issue, you can use: @spy github issues create_comment repo=minitest-distributed issue_id=$NEW_ISSUE_ID on any relevant Slack threads to save that information here.
-
-**Additional context**
-Add any other context about the problem here.
-
-If possible, assign appropriate severity level, priority (high, medium, low) and difficulty (easy, normal, difficult) labels to the issue.
-SEV-1 <24-72h - Critical issue that actively impacts multiple users/products.
-SEV-2 <14 days - Limited impact app or major inconvenience to app users, workarounds available.
-SEV-3 <60 days - Minor issue requiring action, but not affecting a users ability to use the app.
-SEV-4 <180 days - Minimal impact but something the team wants to fix eventually.
-SEV-5 No timeline - Cosmetic issue or bugs, no effect on user's ability to use tooling.

data/.github/dependabot.yml

@@ -1,20 +0,0 @@
-version: 2
-registries:
-  rubygems-server-pkgs-shopify-io:
-    type: rubygems-server
-    url: https://pkgs.shopify.io
-    username: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_USERNAME}}
-    password: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_PASSWORD}}
-  github-com:
-    type: git
-    url: https://github.com
-    username: ${{secrets.DEPENDENCIES_GITHUB_USER}}
-    password: ${{secrets.DEPENDENCIES_GITHUB_TOKEN}}
-updates:
-  - package-ecosystem: bundler
-    directory: "/"
-    schedule:
-      interval: daily
-    open-pull-requests-limit: 100
-    insecure-external-code-execution: allow
-    registries: "*"

data/.github/workflows/ruby.yml

@@ -1,111 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
-# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
-
-name: Ruby
-
-on: push
-
-jobs:
-  test_32:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:3.2
-
-    services:
-      redis:
-        image: redis
-      toxiproxy:
-        image: ghcr.io/shopify/toxiproxy:2.5.0
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Run tests
-      run: bin/rake test
-      env:
-        REDIS_URL: redis://toxiproxy:22220
-        TOXIPROXY_HOST: http://toxiproxy:8474
-
-  test_31:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:3.1
-
-    services:
-      redis:
-        image: redis
-      toxiproxy:
-        image: ghcr.io/shopify/toxiproxy:2.5.0
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Run tests
-      run: bin/rake test
-      env:
-        REDIS_URL: redis://toxiproxy:22220
-        TOXIPROXY_HOST: http://toxiproxy:8474
-
-  test_30:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:3.0
-
-    services:
-      redis:
-        image: redis
-      toxiproxy:
-        image: ghcr.io/shopify/toxiproxy:2.5.0
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Run tests
-      run: bin/rake test
-      env:
-        REDIS_URL: redis://toxiproxy:22220
-        TOXIPROXY_HOST: http://toxiproxy:8474
-
-  test_27:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:2.7
-
-    services:
-      redis:
-        image: redis
-      toxiproxy:
-        image: ghcr.io/shopify/toxiproxy:2.5.0
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Run tests
-      run: bin/rake test
-      env:
-        REDIS_URL: redis://toxiproxy:22220
-        TOXIPROXY_HOST: http://toxiproxy:8474
-
-  typecheck:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:2.7
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Typecheck Ruby code
-      run: bin/srb tc
-
-  lint:
-    runs-on: shopify-ubuntu-latest
-    container: ruby:3.2
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install dependencies
-      run: gem install bundler && bundle install
-    - name: Lint Ruby code
-      run: bin/rubocop