paraspec 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76dfd75abb077976992e37c9934206e610f652f8f7d88564bcfdd9356cf473e1
-  data.tar.gz: 5d08c00a5d951348ba228bb94509158ddae389b21e0e500b011cbdb6aa379a52
+  metadata.gz: d6a3db1f3be0ea3c3b137d2d22ed55d29e402a94f72123c2c5d04eb5c3560569
+  data.tar.gz: 25ea9c3f19e0802ca3db168c0090d1283f5db337477fd42dbcb993f9ccbf2c0a
 SHA512:
-  metadata.gz: e04a18244f08e43e84a9a610405fa6972820f658b84ba67e81a954fd5635873aba6e53ddef8277985d124dbcb6ec6055e5dfbe7f51930c92e314c44078869aeb
-  data.tar.gz: 8b7ef16dee8bb3506442cb8a7b761dbb91929881acd74c4369203f390381de0a3902d59341ce58eb23fa41684754bbd911bb8f67b3ba8a80eddd3fa379689aa4
+  metadata.gz: 6f71a077b52ac683f766e15104204fb27e25c276d08d203f27d3eae7171f5d52ddf4003951f1a8185042d9c382065c2e5d3824719462efb8eb7897063cb978db
+  data.tar.gz: 8cf4dece33abe87139dc218ad1848f30d4ed02b2e60df98005aa7f6c2253b96f42c9eb1b16cfb782ca903328f275c6823a83c737277df1c946956c6055928bdb

data/README.md

@@ -18,7 +18,8 @@ test by test basis, avoiding interleaving output of different tests together.
 This output capture can be performed for output generated by C extensions
 as well as plain Ruby code.
 4. Test results are seamlessly integrated by the master, such that
-a parallel run produces a single progress bar with Fuubar across all workers.
+a parallel run produces a single progress bar with
+[Fuubar](https://github.com/thekompanee/fuubar) across all workers.
 5. Test grouping: paraspec can[**] group the tests by file, top level example
 group, bottom level example group and individually by examples.
 Larger groupings reduce overhead but limit concurrency.
@@ -33,6 +34,27 @@ is not currently on the roadmap.
 
 [**] Currently only grouping by bottom level example group is implemented.
 
+## Performance
+
+How much of a difference does paraspec make? The answer, as one might
+expect, varies greatly with the test suite being run as well as available
+hardware. Here are some examples:
+
+| Example | Hardware | Sequential | Paraspec (c=2) | Paraspec (c=4) |
+|---------|------------|----------------|----------------|----------|
+| [MongoDB Ruby Driver](https://docs.mongodb.com/ruby-driver/current/) test suite | Travis CI | 16 minutes | 13 minutes | 10-11 minutes |
+| [MongoDB Ruby Driver](https://docs.mongodb.com/ruby-driver/current/) test suite | 14-core workstation | 15 minutes | | 4 minutes |
+
+[Exampe Travis build](https://travis-ci.org/p-mongo/mongo-ruby-driver-paraspec/builds/411986888)
+
+Even on Travis, which is likely limited to a single core, using 4x concurrency
+reduces the runtime by 5 minutes. On a developer workstation which doesn't
+download binaries on every test run the speedup is closer to linear.
+Waiting 4 minutes instead of 15 for a complete test suite means the engineers
+can actually run the complete test suite as part of their normal workflow,
+instead of sending the code to a CI platform and context switching to
+a different project.
+
 ## Usage
 
 Add paraspec to your Gemfile:
@@ -88,7 +110,8 @@ failing tests.
 
 ### Debugging
 
-Paraspec offers several debugging aids. The first one is the terminal option:
+Paraspec offers several debugging aids. For interactive debugging use the
+terminal option:
 
     paraspec -T
 
@@ -108,6 +131,34 @@ The debugging output is turned on with `-d`/`--debug` option:
     paraspec -d ipc     # IPC requests and responses
     paraspec -d perf    # timing & performance information
 
+### Executing Tests Together
+
+It is possible to specify that a group of tests should be executed in the
+same worker rather than distributed. This is useful when the setup for
+the tests is expensive and therefore is done once with multiple tests
+defined on the result.
+
+The grouping is defined by specifying `{group: true}` paraspec option on
+a `describe` or a `context` block as follows:
+
+    describe 'Run these together', paraspec: {group: true} do
+      before(:all) do
+        # expensive setup
+      end
+      
+      it 'does something' do
+        # ...
+      end
+      
+      it 'does something else' do
+        # ...
+      end
+      
+      after(:all) do
+        # teardown
+      end
+    end
+
 ## Caveats
 
 The master and workers need to all define the same example groups and

data/bin/paraspec

@@ -16,7 +16,7 @@ OptionParser.new do |opts|
     options[:concurrency] = v.to_i
   end
 
-  opts.on('-d', '--debug=SUBSYSTEM', 'Output debugging information for SUBSYSTEM') do |v|
+  opts.on('-d', '--debug=SUBSYSTEM', 'Output debugging information for SUBSYSTEM (state/ipc/perf)') do |v|
     options[:"debug_#{v}"] = true
     options[:debug] = true
   end
@@ -44,7 +44,7 @@ end
 RSpec.configuration.files_or_directories_to_run = files
 =end
 
-%w(ipc state perf).each do |subsystem|
+%w(state ipc perf).each do |subsystem|
   if options.delete(:"debug_#{subsystem}")
     Paraspec.logger.send("log_#{subsystem}=", true)
   end

data/lib/paraspec.rb

@@ -1,5 +1,6 @@
 require 'paraspec/rspec_patches'
 require 'paraspec/version'
+require 'paraspec/errors'
 require 'paraspec/logger'
 require 'paraspec/ipc'
 require 'paraspec/drb_helpers'

data/lib/paraspec/errors.rb

@@ -0,0 +1,13 @@
+module Paraspec
+  # Base class for all Paraspec errors
+  class Error < StandardError; end
+
+  # Different set of examples between master and a worker
+  class InconsistentTestSuite < Error; end
+
+  # A worker process exited with non-zero status
+  class WorkerFailed < StandardError; end
+
+  # Master process exited with non-zero status
+  class MasterFailed < StandardError; end
+end

data/lib/paraspec/http_client.rb

@@ -6,7 +6,7 @@ module Paraspec
     def initialize(options={})
       @terminal = options[:terminal]
 
-      @client = Faraday.new(url: "http://localhost:#{Paraspec::MASTER_APP_PORT}") do |client|
+      @client = Faraday.new(url: "http://localhost:#{Paraspec::Ipc.master_app_port}") do |client|
         client.adapter :net_http
         if @terminal
           client.options.timeout = 100000

data/lib/paraspec/ipc.rb

@@ -1,11 +1,26 @@
 require 'hashie'
+require 'socket'
 
 module Paraspec
-  #SUPERVISOR_DRB_URI = "druby://localhost:6030"
-  MASTER_DRB_URI = "druby://localhost:6031"
-  MASTER_APP_PORT = 6031
-
   class IpcHash < Hash
     include Hashie::Extensions::IndifferentAccess
   end
+
+  module Ipc
+    class << self
+      def pick_master_app_port
+        port = 10000 + rand(40000)
+        begin
+          server = TCPServer.new('127.0.0.1', port)
+          server.close
+          return @master_app_port = port
+        rescue Errno::EADDRINUSE
+          port = 10000 + rand(40000)
+          retry
+        end
+      end
+
+      attr_reader :master_app_port
+    end
+  end
 end

data/lib/paraspec/master.rb

@@ -58,7 +58,7 @@ module Paraspec
       end
       @queue = []
       if @non_example_exception_count == 0
-        @queue += RSpecFacade.all_example_groups
+        @queue += RSpecFacade.queueable_example_groups
         puts "#{@queue.length} example groups queued"
       else
         puts "#{@non_example_exception_count} errors outside of examples, aborting"
@@ -104,13 +104,20 @@ module Paraspec
         example_group = @queue.shift
         return nil if example_group.nil?
 
-        # TODO I am still not 100% on what should be filtered and pruned where,
-        # but we shouldn't be returning a specification here unless
-        # there are tests in it that a worker will run
-        pruned_examples = RSpec.configuration.filter_manager.prune(example_group.examples)
-        next if pruned_examples.empty?
+        if example_group.metadata[:paraspec] && example_group.metadata[:paraspec][:group]
+          # unsplittable example group
+          # pass to worker as is and have the worker prune examples
+          m = example_group.metadata
+        else
+          # TODO I am still not 100% on what should be filtered and pruned where,
+          # but we shouldn't be returning a specification here unless
+          # there are tests in it that a worker will run
+          pruned_examples = RSpec.configuration.filter_manager.prune(example_group.examples)
+          next if pruned_examples.empty?
+
+          m = example_group.metadata
+        end
 
-        m = example_group.metadata
         return {
           file_path: m[:file_path],
           scoped_id: m[:scoped_id],

data/lib/paraspec/msgpack_client.rb

@@ -39,7 +39,7 @@ module Paraspec
     private def connect
       start_time = Time.now
       begin
-        @socket = TCPSocket.new('127.0.0.1', MASTER_APP_PORT)
+        @socket = TCPSocket.new('127.0.0.1', Paraspec::Ipc.master_app_port)
       rescue Errno::ECONNREFUSED
         if !@terminal && Time.now - start_time > DrbHelpers::WAIT_TIME
           raise

data/lib/paraspec/msgpack_server.rb

@@ -8,10 +8,11 @@ module Paraspec
 
     def initialize(master)
       @master = master
+      @lock = Mutex.new
     end
 
     def run
-      @socket = ::TCPServer.new('127.0.0.1', MASTER_APP_PORT)
+      @socket = ::TCPServer.new('127.0.0.1', Paraspec::Ipc.master_app_port)
       begin
         while true
           s = @socket.accept_nonblock
@@ -29,30 +30,46 @@ module Paraspec
       Thread.new do
         u = unpacker(s)
         u.each do |obj|
-          result = nil
-          time = Benchmark.realtime do
-            action = obj['action'].gsub('-', '_')
-            payload = obj['payload']
-            if payload
-              payload = IpcHash.new.merge(payload)
-              args = [payload]
-            else
-              args = []
-            end
+          # Apparently this block may be run concurrently, which we are not
+          # equipped for.
+          @lock.synchronize do
+            process_request(s, obj)
+          end
+        end
+      end
+    end
 
-            Paraspec.logger.debug_ipc("SrvReq:#{obj['id']} #{obj}")
-            result = @master.send(action, *args)
+    def process_request(s, obj)
+      result = nil
+      time = Benchmark.realtime do
+        action = obj['action'].gsub('-', '_')
+        payload = obj['payload']
+        if payload
+          payload = IpcHash.new.merge(payload)
+          args = [payload]
+        else
+          args = []
+        end
 
-            pk = packer(s)
-            resp = {result: result}
-            Paraspec.logger.debug_ipc("SrvRes:#{obj['id']} #{resp}")
-            pk.write(resp)
-            pk.flush
-            s.flush
-          end
-          Paraspec.logger.debug_perf("SrvReq:#{obj['id']} #{obj['action']}: #{result} #{'%.3f msec' % (time*1000)}")
+        Paraspec.logger.debug_ipc("SrvReq:#{obj['id']} #{obj}")
+        result = @master.send(action, *args)
+
+        pk = packer(s)
+        resp = {result: result}
+        Paraspec.logger.debug_ipc("SrvRes:#{obj['id']} #{resp}")
+        if payload && payload['_noret']
+          # Requester does not want a response.
+          # We return a hash here because deserializer expects a hash.
+          # Always returning `resp` is problematic because it can
+          # be non-serializable.
+          pk.write({})
+        else
+          pk.write(resp)
         end
+        pk.flush
+        s.flush
       end
+      Paraspec.logger.debug_perf("SrvReq:#{obj['id']} #{obj['action']}: #{result} #{'%.3f msec' % (time*1000)}")
     end
   end
 end

data/lib/paraspec/rspec_facade.rb

@@ -7,7 +7,27 @@ module Paraspec
 
     class << self
       extend Forwardable
-      def_delegators :instance, :all_example_groups, :all_examples
+      def_delegators :instance, :queueable_example_groups, :all_example_groups, :all_examples
+    end
+
+    def queueable_example_groups
+      @queueable_example_groups ||= begin
+        groups = [] + RSpec.world.example_groups
+        all_groups = []
+        until groups.empty?
+          new_groups = []
+          groups.each do |group|
+            all_groups << group
+            if group.metadata[:paraspec] && group.metadata[:paraspec][:group]
+              # unsplittable group
+            else
+              new_groups += group.children
+            end
+          end
+          groups = new_groups
+        end
+        all_groups
+      end
     end
 
     def all_example_groups

data/lib/paraspec/supervisor.rb

@@ -21,6 +21,14 @@ module Paraspec
         Process.setpgrp
       end
 
+      at_exit do
+        # It seems that when the tests are run in Travis, killing
+        # paraspec vaporizes the standard output... flush the
+        # standard output streams here to work around.
+        STDERR.flush
+        STDOUT.flush
+      end
+
       supervisor_pid = $$
       at_exit do
         # We fork, therefore this handler will be run in master and
@@ -41,9 +49,11 @@ module Paraspec
         end
       end
 
+      Paraspec::Ipc.pick_master_app_port
+
       rd, wr = IO.pipe
       if @master_pid = fork
-        # parent
+        # parent - supervisor
         wr.close
         @master_pipe = rd
         run_supervisor
@@ -93,7 +103,7 @@ module Paraspec
         Paraspec.logger.debug_state("Waiting for workers")
         @worker_pids.each_with_index do |pid, i|
           Paraspec.logger.debug_state("Waiting for worker #{i+1} at #{pid}")
-          wait_for_process(pid)
+          wait_for_process(pid, "Worker #{i+1}", WorkerFailed)
         end
         status = 0
       else
@@ -108,15 +118,14 @@ module Paraspec
       end
       Paraspec.logger.debug_state("Asking master to stop")
       master_client.request('stop')
-      wait_for_process(@master_pid)
+      wait_for_process(@master_pid, 'Master', MasterFailed)
       exit status
     end
 
-    def wait_for_process(pid)
-      begin
-        Process.wait(pid)
-      rescue Errno::ECHILD
-        # already dead
+    def wait_for_process(pid, process_name, exception_class)
+      pid, status = Process.wait2(pid)
+      if status.exitstatus != 0
+        raise exception_class, "#{process_name} exited with status #{status.exitstatus}"
       end
     end
 

data/lib/paraspec/version.rb

@@ -1,3 +1,3 @@
 module Paraspec
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/paraspec/worker.rb

@@ -46,7 +46,7 @@ module Paraspec
         # being run.
         puts "Worker #{@number} has #{RSpecFacade.all_examples.count} examples, master has #{master_example_count}"
         #byebug
-        raise "Worker #{@number} has #{RSpecFacade.all_examples.count} examples, master has #{master_example_count}"
+        raise InconsistentTestSuite, "Worker #{@number} has #{RSpecFacade.all_examples.count} examples, master has #{master_example_count}"
       end
 
       while true

data/lib/paraspec/worker_formatter.rb

@@ -54,7 +54,7 @@ module Paraspec
         file_path: notification.example.metadata[:file_path],
         scoped_id: notification.example.metadata[:scoped_id],
       }
-      @master_client.request('notify-example-started', spec: spec)
+      @master_client.request('notify-example-started', spec: spec, _noret: true)
     end
 
     def example_passed(notification)
@@ -68,7 +68,7 @@ module Paraspec
       }
       execution_result = notification.example.execution_result
       @master_client.request('example-passed',
-        spec: spec, result: execution_result)
+        spec: spec, result: execution_result, _noret: true)
     end
 
     def example_failed(notification)

data/lib/paraspec/worker_runner.rb

@@ -29,14 +29,30 @@ module Paraspec
       #byebug
         raise "No example group for #{spec.inspect}"
       end
-      examples = group.examples
-      #Paraspec.logger.debug_state("Spec #{spec}: #{examples.length} examples")
-      return if examples.empty?
-      ids = examples.map { |e| e.metadata[:scoped_id] }
-      RSpec.configuration.send(:instance_variable_set, '@filter_manager', RSpec::Core::FilterManager.new)
-      RSpec.configuration.filter_manager.add_ids(spec[:file_path], ids)
-      RSpec.world.filter_examples
-      examples = RSpec.configuration.filter_manager.prune(examples)
+      if group.metadata[:paraspec] && group.metadata[:paraspec][:group]
+        # unsplittable group
+        # get all examples in child groups
+        examples = []
+        group_queue = [group]
+        until group_queue.empty?
+          next_group_queue = []
+          group_queue.each do |group|
+            next_group_queue += group.children
+            examples += group.examples
+          end
+          group_queue = next_group_queue
+        end
+      else
+        # leaf group
+        examples = group.examples
+        #Paraspec.logger.debug_state("Spec #{spec}: #{examples.length} examples")
+        return if examples.empty?
+        ids = examples.map { |e| e.metadata[:scoped_id] }
+        RSpec.configuration.send(:instance_variable_set, '@filter_manager', RSpec::Core::FilterManager.new)
+        RSpec.configuration.filter_manager.add_ids(spec[:file_path], ids)
+        RSpec.world.filter_examples
+        examples = RSpec.configuration.filter_manager.prune(examples)
+      end
       return if examples.empty?
       # It is important to run the entire world here because if
       # a particular example group is run, before/after :all hooks

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: paraspec
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Oleg Pudeyev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-04 00:00:00.000000000 Z
+date: 2019-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.7.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.7.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: childprocess
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +84,7 @@ files:
 - bin/paraspec
 - lib/paraspec.rb
 - lib/paraspec/drb_helpers.rb
+- lib/paraspec/errors.rb
 - lib/paraspec/http_client.rb
 - lib/paraspec/http_server.rb
 - lib/paraspec/ipc.rb
@@ -115,9 +122,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
-summary: paraspec-0.0.2
+summary: paraspec-0.0.3
 test_files: []