test-queue 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 550cb255057332f8736ac60ed578044f7ec94694
-  data.tar.gz: c1e268f35f59f1eae129e2f4c6df801ac7bd32e5
+  metadata.gz: 14379da7829bc7f9e40a46dce33c344ff0bd434a
+  data.tar.gz: 7b21acc430c9bcada801f9aa48352afffdd78708
 SHA512:
-  metadata.gz: 7dc83049443c7c45cbb8bd1a9a72316ce22d674caf7339af6943a15ddf8c2fbad6eec9160cf36245d0058ac645b10ea72b803ccd7f422c2bda91d9c863aa8601
-  data.tar.gz: e68b2b79d5b981e054d1b3f24dab94a060983d4ca0a7d8861f5ef2b8286c2249fb142e2a3ec7f69437b89ca7620fdbd3dc73c1dc59f9ae0036ad5fd15470f947
+  metadata.gz: ac7142497a109a2c40ce726e74f32d5f86c18c21d6618820f85b3d13986c39214f4b6a1feeb3eb73d53ec4d93bb382a7b68e123ebb7502a0a6f2e49fe6062258
+  data.tar.gz: 3a9bfaee682cbde11361547ce43d4a7b6c61f60a8f31b8be9ea2081aec533741281fc6eec4ddcb9b3eb7c7186bfede9f5ea4bbcdaba2f27cbccda2fb785e51fa

data/Gemfile-cucumber1-3.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-cucumber2-4.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-minitest4.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-minitest5.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-rspec2-1.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-rspec3-0.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-rspec3-1.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-rspec3-2.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile-testunit.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    test-queue (0.3.1)
+    test-queue (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/lib/test_queue/iterator.rb

@@ -2,7 +2,7 @@ module TestQueue
   class Iterator
     attr_reader :sock
 
-    def initialize(test_framework, sock, filter=nil, early_failure_limit: nil)
+    def initialize(test_framework, sock, filter=nil, run_token:, early_failure_limit: nil)
       @test_framework = test_framework
       @done = false
       @suite_stats = []
@@ -14,6 +14,7 @@ module TestQueue
       end
       @failures = 0
       @early_failure_limit = early_failure_limit
+      @run_token = run_token
     end
 
     def each
@@ -29,7 +30,7 @@ module TestQueue
           connect_to_master("KABOOM")
           break
         else
-          client = connect_to_master('POP')
+          client = connect_to_master("POP #{Socket.gethostname} #{Process.pid}")
         end
         break if client.nil?
         _r, _w, e = IO.select([client], nil, [client], nil)
@@ -79,6 +80,7 @@ module TestQueue
         else
           UNIXSocket.new(@sock)
         end
+      sock.puts("TOKEN=#{@run_token}")
       sock.puts(cmd)
       sock
     rescue Errno::EPIPE

data/lib/test_queue/runner.rb

@@ -31,6 +31,8 @@ module TestQueue
     attr_accessor :concurrency, :exit_when_done
     attr_reader :stats
 
+    TOKEN_REGEX = /^TOKEN=(\w+)/
+
     def initialize(test_framework, concurrency=nil, socket=nil, relay=nil)
       @test_framework = test_framework
       @stats = Stats.new(stats_file)
@@ -63,7 +65,7 @@ module TestQueue
         @queue.sort_by! { |suite_name, path| @whitelist.index(suite_name) }
       end
 
-      @awaited_suites = Set.new(@whitelist - @queue.map(&:first))
+      @awaited_suites = Set.new(@whitelist)
       @original_queue = Set.new(@queue).freeze
 
       @workers = {}
@@ -72,7 +74,7 @@ module TestQueue
       @concurrency =
         concurrency ||
         (ENV['TEST_QUEUE_WORKERS'] && ENV['TEST_QUEUE_WORKERS'].to_i) ||
-        if File.exists?('/proc/cpuinfo')
+        if File.exist?('/proc/cpuinfo')
           File.read('/proc/cpuinfo').split("\n").grep(/processor/).size
         elsif RUBY_PLATFORM =~ /darwin/
           `/usr/sbin/sysctl -n hw.activecpu`.to_i
@@ -107,7 +109,12 @@ module TestQueue
         @queue = []
       end
 
+      @discovered_suites = Set.new
+      @assignments = {}
+
       @exit_when_done = true
+
+      @aborting = false
     end
 
     # Run the tests.
@@ -134,9 +141,23 @@ module TestQueue
       puts "==> Summary (#{@completed.size} workers in %.4fs)" % (Time.now-@start_time)
       puts
 
+      estatus = 0
+      misrun_suites = []
+      unassigned_suites = []
       @failures = ''
       @completed.each do |worker|
+        estatus += worker.status.exitstatus
         @stats.record_suites(worker.suites)
+        worker.suites.each do |suite|
+          assignment = @assignments.delete([suite.name, suite.path])
+          host = worker.host || Socket.gethostname
+          if assignment.nil?
+            unassigned_suites << [suite.name, suite.path]
+          elsif assignment != [host, worker.pid]
+            misrun_suites << [suite.name, suite.path] + assignment + [host, worker.pid]
+          end
+          @discovered_suites.delete([suite.name, suite.path])
+        end
 
         summarize_worker(worker)
 
@@ -160,15 +181,44 @@ module TestQueue
         puts @failures
       end
 
+      if !relay?
+        unless @discovered_suites.empty?
+          estatus += 1
+          puts
+          puts "The following suites were discovered but were not run:"
+          puts
+
+          @discovered_suites.sort.each do |suite_name, path|
+            puts "#{suite_name} - #{path}"
+          end
+        end
+        unless unassigned_suites.empty?
+          estatus += 1
+          puts
+          puts "The following suites were not discovered but were run anyway:"
+          puts
+          unassigned_suites.sort.each do |suite_name, path|
+            puts "#{suite_name} - #{path}"
+          end
+        end
+        unless misrun_suites.empty?
+          estatus += 1
+          puts
+          puts "The following suites were run on the wrong workers:"
+          puts
+          misrun_suites.each do |suite_name, path, target_host, target_pid, actual_host, actual_pid|
+            puts "#{suite_name} - #{path}: #{actual_host} (#{actual_pid}) - assigned to #{target_host} (#{target_pid})"
+          end
+        end
+      end
+
       puts
 
       @stats.save
 
       summarize
 
-      estatus = @completed.inject(0){ |s, worker| s + worker.status.exitstatus }
-      estatus = 255 if estatus > 255
-      estatus
+      [estatus, 255].min
     end
 
     def summarize
@@ -201,7 +251,7 @@ module TestQueue
           @socket = "#$1:#$2"
           @server = TCPServer.new(address, port)
         else
-          FileUtils.rm(@socket) if File.exists?(@socket)
+          FileUtils.rm(@socket) if File.exist?(@socket)
           @server = UNIXServer.new(@socket)
         end
       end
@@ -217,7 +267,8 @@ module TestQueue
       sock = connect_to_relay
       message = @slave_message ? " #{@slave_message}" : ""
       message.gsub!(/(\r|\n)/, "") # Our "protocol" is newline-separated
-      sock.puts("SLAVE #{@concurrency} #{Socket.gethostname} #{@run_token}#{message}")
+      sock.puts("TOKEN=#{@run_token}")
+      sock.puts("SLAVE #{@concurrency} #{Socket.gethostname} #{message}")
       response = sock.gets.strip
       unless response == "OK"
         STDERR.puts "*** Got non-OK response from master: #{response}"
@@ -245,7 +296,7 @@ module TestQueue
         pid = fork do
           @server.close if @server
 
-          iterator = Iterator.new(@test_framework, relay?? @relay : @socket, method(:around_filter), early_failure_limit: @early_failure_limit)
+          iterator = Iterator.new(@test_framework, relay?? @relay : @socket, method(:around_filter), early_failure_limit: @early_failure_limit, run_token: @run_token)
           after_fork_internal(num, iterator)
           ret = run_worker(iterator) || 0
           cleanup_worker
@@ -276,6 +327,7 @@ module TestQueue
             Kernel.exit!(0) if terminate
 
             @server.connect_address.connect do |sock|
+              sock.puts("TOKEN=#{@run_token}")
               sock.puts("NEW SUITE #{Marshal.dump([suite_name, path])}")
             end
           end
@@ -305,8 +357,11 @@ module TestQueue
         return
       end
 
+      @discovered_suites << [suite_name, path]
+
       if @original_queue.include?([suite_name, path])
         # This suite was already added to the queue some other way.
+        @awaited_suites.delete(suite_name)
         return
       end
 
@@ -393,12 +448,12 @@ module TestQueue
     end
 
     def collect_worker_data(worker)
-      if File.exists?(file = "/tmp/test_queue_worker_#{worker.pid}_output")
+      if File.exist?(file = "/tmp/test_queue_worker_#{worker.pid}_output")
         worker.output = IO.binread(file)
         FileUtils.rm(file)
       end
 
-      if File.exists?(file = "/tmp/test_queue_worker_#{worker.pid}_suites")
+      if File.exist?(file = "/tmp/test_queue_worker_#{worker.pid}_suites")
         worker.suites.replace(Marshal.load(IO.binread(file)))
         FileUtils.rm(file)
       end
@@ -426,29 +481,38 @@ module TestQueue
           reap_workers(false) # check for worker deaths
         else
           sock = @server.accept
+          token = sock.gets.strip
           cmd = sock.gets.strip
+
+          token = token[TOKEN_REGEX, 1]
+          # If we have a slave from a different test run, respond with "WRONG RUN", and it will consider the test run done.
+          if token != @run_token
+            message = token.nil? ? "Worker sent no token to master" : "Worker from run #{token} connected to master"
+            STDERR.puts "*** #{message} for run #{@run_token}; ignoring."
+            sock.write("WRONG RUN\n")
+            next
+          end
+
           case cmd
-          when /^POP/
+          when /^POP (\S+) (\d+)/
             # If we have a slave from a different test run, don't respond, and it will consider the test run done.
+            hostname = $1
+            pid = Integer($2)
             if awaiting_suites?
               sock.write(Marshal.dump("WAIT"))
             elsif obj = @queue.shift
               data = Marshal.dump(obj)
               sock.write(data)
+              @assignments[obj] = [hostname, pid]
             end
-          when /^SLAVE (\d+) ([\w\.-]+) (\w+)(?: (.+))?/
+          when /^SLAVE (\d+) ([\w\.-]+)(?: (.+))?/
             num = $1.to_i
             slave = $2
-            run_token = $3
-            slave_message = $4
-            if run_token == @run_token
-              # If we have a slave from a different test run, don't respond, and it will consider the test run done.
-              sock.write("OK\n")
-              remote_workers += num
-            else
-              STDERR.puts "*** Worker from run #{run_token} connected to master for run #{@run_token}; ignoring."
-              sock.write("WRONG RUN\n")
-            end
+            slave_message = $3
+
+            sock.write("OK\n")
+            remote_workers += num
+
             message = "*** #{num} workers connected from #{slave} after #{Time.now-@start_time}s"
             message << " " + slave_message if slave_message
             STDERR.puts message
@@ -464,6 +528,8 @@ module TestQueue
             # worker reporting an abnormal number of test failures;
             # stop everything immediately and report the results.
             break
+          else
+            STDERR.puts("Ignoring unrecognized command: \"#{cmd}\"")
           end
           sock.close
         end
@@ -498,6 +564,7 @@ module TestQueue
       data = Marshal.dump(worker)
 
       sock = connect_to_relay
+      sock.puts("TOKEN=#{@run_token}")
       sock.puts("WORKER #{data.bytesize}")
       sock.write(data)
     ensure

data/lib/test_queue/stats.rb

@@ -74,7 +74,7 @@ module TestQueue
     def load
       data = begin
                File.open(@path, "rb") { |f| Marshal.load(f) }
-             rescue Errno::ENOENT, EOFError, TypeError
+             rescue Errno::ENOENT, EOFError, TypeError, ArgumentError
              end
       return unless data && data.is_a?(Hash) && data[:version] == CURRENT_VERSION
       data[:suites].each do |suite_hash|

data/test-queue.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'test-queue'
-  s.version = '0.4.0'
+  s.version = '0.4.1'
   s.summary = 'parallel test runner'
   s.description = 'minitest/rspec parallel test runner for CI environments'
 

data/test/minitest5.bats

@@ -74,23 +74,52 @@ assert_test_queue_force_ordering() {
   assert_output_contains "Failed to discover DoesNotExist specified in TEST_QUEUE_FORCE"
 }
 
-@test "multi-master succeeds when all tests pass" {
+@test "multi-master central master succeeds when all tests pass" {
   export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
-  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
+  export SLEEP_AS_RELAY=1
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
   sleep 0.1
-  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
   wait
 
   assert_status 0
   assert_output_contains "Starting test-queue master"
 }
 
-@test "multi-master fails when a test fails" {
+@test "multi-master remote master succeeds when all tests pass" {
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  export SLEEP_AS_MASTER=1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "multi-master central master fails when a test fails" {
+  export FAIL=1
+  export SLEEP_AS_RELAY=1
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "multi-master remote master fails when a test fails" {
   export FAIL=1
+  export SLEEP_AS_MASTER=1
   export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
-  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
   sleep 0.1
-  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
   wait
 
   assert_status 1

data/test/rspec.bats

@@ -37,7 +37,7 @@ setup() {
   assert_output_contains "0 examples, 0 failures"
 }
 
-@test "Use to Shared Exsamples." {
+@test "rspec-queue supports shared example groups" {
   run bundle exec rspec-queue ./test/samples/sample_use_shared_example1_spec.rb \
                               ./test/samples/sample_use_shared_example2_spec.rb
   assert_status 0

data/test/samples/sample_use_shared_example1_spec.rb

@@ -1,7 +1,7 @@
 require 'rspec'
 require_relative 'sample_rspec_helper'
 
-describe 'Use SharedExcaplesFor test_1' do
+describe 'Use SharedExamplesFor test_1' do
   subject { 5 }
   it_behaves_like 'Sample Example'
 end

data/test/samples/sample_use_shared_example2_spec.rb

@@ -1,7 +1,7 @@
 require 'rspec'
 require_relative 'sample_rspec_helper'
 
-describe 'Use SharedExcaplesFor test_2' do
+describe 'Use SharedExamplesFor test_2' do
   subject { 5 }
   it_behaves_like 'Sample Example'
 end

data/test/sleepy_runner.rb

@@ -0,0 +1,14 @@
+require 'test_queue'
+require 'test_queue/runner/minitest'
+
+class SleepyTestRunner < TestQueue::Runner::MiniTest
+  def after_fork(num)
+    if ENV['SLEEP_AS_RELAY'] && relay? 
+      sleep 5
+    elsif ENV['SLEEP_AS_MASTER'] && !relay?
+      sleep 5
+    end
+  end
+end
+
+SleepyTestRunner.new.execute

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: test-queue
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Aman Gupta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-04 00:00:00.000000000 Z
+date: 2017-01-06 00:00:00.000000000 Z
 dependencies: []
 description: minitest/rspec parallel test runner for CI environments
 email: ruby@tmm1.net
@@ -86,6 +86,7 @@ files:
 - test/samples/sample_testunit.rb
 - test/samples/sample_use_shared_example1_spec.rb
 - test/samples/sample_use_shared_example2_spec.rb
+- test/sleepy_runner.rb
 - test/testlib.bash
 - test/testunit.bats
 homepage: http://github.com/tmm1/test-queue