parallel_cucumber 0.2.16 → 0.2.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3754f455d081d50ef24488f466b9d15da6f40fb4a55b956cf2a2bbef0aea127b
-  data.tar.gz: dfb977764ba2abc9a8839e9f15ac158adbe066ac23f1e1b60b896f2b8b59f9ad
+  metadata.gz: 5664f82c63144d99bec9dc1880905601d0de45b2431b929eee77c0c4abb4521f
+  data.tar.gz: 9c5fce5710b7a36270c5f84431bc32195bf54a37779380ea584e04f892b8e616
 SHA512:
-  metadata.gz: 0fd9d4d21114405cd8b166838b300455f29b6911ab8bc7c717c0d4b0d041b6e7753726e4a3a8350e73054a4c90b231348eff6f970fb31c1123c163fc74532c76
-  data.tar.gz: 506ec27ac0b9ba080e1e904ad70335c78dc0be6c17143d873d61babcc30381c087773510f3a9f600b9110484a14c5cf9d805395c593a4f4153df01295f5e167e
+  metadata.gz: c51f9ebe8c466e9e31936ddf59bc1944f5ddab128e317d4185b4047d7d7c2f63f091c242556ac2185bd67bbd3de7145fe31fbb3fc51239e56b1b046b3412087e
+  data.tar.gz: bac0d600b9e52aacc815a7ad2e3138470c5efafa2aad96d4c94ba82bae7d82df2bdd48ce21b2b78b09c98045f1f40e3e13b760ef371a8f16babd7bfdeef180c4

data/lib/parallel_cucumber.rb

@@ -7,8 +7,10 @@ require 'parallel_cucumber/helper/processes'
 require 'parallel_cucumber/helper/queue'
 require 'parallel_cucumber/helper/utils'
 require 'parallel_cucumber/hooks'
+require 'parallel_cucumber/job'
 require 'parallel_cucumber/logger'
 require 'parallel_cucumber/main'
 require 'parallel_cucumber/status'
 require 'parallel_cucumber/version'
+require 'parallel_cucumber/worker_manager'
 require 'parallel_cucumber/worker'

data/lib/parallel_cucumber/cli.rb

@@ -19,7 +19,8 @@ module ParallelCucumber
       n: 0, # Default: computed from longest list in json parameters, minimum 1.
       queue_connection_params: ['redis://127.0.0.1:6379', DateTime.now.strftime('queue-%Y%m%d%H%M%S')],
       worker_delay: 0,
-      test_command: 'cucumber'
+      test_command: 'cucumber',
+      backup_worker_count: 0
     }.freeze
 
     def initialize(argv)
@@ -61,6 +62,10 @@ module ParallelCucumber
           options[:n] = n
         end
 
+        opts.on('--backup-worker-count BACKUP_WORKERS', Integer, 'How many free workers to hold before all tasks are done. Default is none') do |n|
+          options[:backup_worker_count] = n
+        end
+
         opts.on('-o', '--cucumber-options "OPTIONS"', 'Run cucumber with these options') do |cucumber_options|
           options[:cucumber_options] = cucumber_options
         end

data/lib/parallel_cucumber/job.rb

@@ -0,0 +1,13 @@
+module ParallelCucumber
+  class Job
+    RUN_TESTS = :run_tests
+    PRECHECK  = :precheck
+    DIE       = :die
+
+    attr_accessor :type, :details
+    def initialize(type, details = nil)
+      @type = type
+      @details = details
+    end
+  end
+end

data/lib/parallel_cucumber/main.rb

@@ -149,17 +149,11 @@ module ParallelCucumber
       Helper::Command.wrap_block(@options[:log_decoration],
                                  @options[:log_decoration]['worker_block'] || 'workers',
                                  @logger) do
-        remaining = (0...number_of_workers).to_a
-        map = Parallel.map(
-          remaining.dup,
-          in_threads: number_of_workers,
-          finish: -> (_, ix, _) { @logger.synch { |l| l.info("Finished: #{ix} remaining: #{remaining -= [ix]}") } }
-        ) do |index|
-          ParallelCucumber::Worker
-            .new(@options, index, @logger)
-            .start(env_for_worker(@options[:env_variables], index))
-        end
-        map.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
+
+        worker_manager =  ParallelCucumber::WorkerManager.new(@options, @logger)
+        worker_manager.start(number_of_workers)
+      ensure
+        worker_manager.kill
       end
     end
 
@@ -169,11 +163,11 @@ module ParallelCucumber
         @logger.info("Inferred worker count #{@options[:n]} from env_variables option")
       end
 
-      number_of_workers = [@options[:n], count].min
+      number_of_workers = [@options[:n], [@options[:backup_worker_count], count].max].min
       unless number_of_workers == @options[:n]
         @logger.info(<<-LOG)
           Number of workers was overridden to #{number_of_workers}.
-          More workers (#{@options[:n]}) requested than tests (#{count})".
+          More workers (#{@options[:n]}) requested than tests (#{count}). BackupWorkerCount: #{@options[:backup_worker_count]}".
         LOG
       end
 
@@ -192,30 +186,5 @@ module ParallelCucumber
       end
       number_of_workers
     end
-
-    private
-
-    def env_for_worker(env_variables, worker_number)
-      env = env_variables.map do |k, v|
-        case v
-        when String, Numeric, TrueClass, FalseClass
-          [k, v]
-        when Array
-          [k, v[worker_number]]
-        when Hash
-          value = v[worker_number.to_s]
-          [k, value] unless value.nil?
-        when NilClass
-        else
-          raise("Don't know how to set '#{v}'<#{v.class}> to the environment variable '#{k}'")
-        end
-      end.compact.to_h
-
-      # Defaults, if absent in env. Shame 'merge' isn't something non-commutative like 'adopts/defaults'.
-      env = { TEST: 1, TEST_PROCESS_NUMBER: worker_number, WORKER_INDEX: worker_number }.merge(env)
-
-      # Overwrite this if it exists in env.
-      env.merge(PARALLEL_CUCUMBER_EXPORTS: env.keys.join(',')).map { |k, v| [k.to_s, v.to_s] }.to_h
-    end
   end
 end

data/lib/parallel_cucumber/version.rb

@@ -1,3 +1,3 @@
 module ParallelCucumber
-  VERSION = '0.2.16'.freeze
+  VERSION = '0.2.17'.freeze
 end

data/lib/parallel_cucumber/worker.rb

@@ -3,28 +3,10 @@ require 'timeout'
 require 'tmpdir' # I loathe Ruby.
 
 module ParallelCucumber
-  class Tracker
-    def initialize(queue)
-      @queue = queue
-    end
-
-    def status
-      queue_length = @queue.length
-      now = Time.now
-      @full ||= queue_length
-      @start ||= now
-      completed = @full - queue_length
-      elapsed = now - @start
-      estimate = (completed == 0) ? '' : " #{(elapsed * @full / completed).to_i}s est"
-      "#{queue_length}/#{@full} left #{elapsed.to_i}s worker#{estimate}"
-    end
-  end
-
   class Worker
     include ParallelCucumber::Helper::Utils
 
-    def initialize(options, index, stdout_logger)
-      @batch_size = options[:batch_size]
+    def initialize(options:, index:, stdout_logger:, manager:)
       @group_by = options[:group_by]
       @batch_timeout = options[:batch_timeout]
       @batch_error_timeout = options[:batch_error_timeout]
@@ -34,7 +16,7 @@ module ParallelCucumber
       @test_command = options[:test_command]
       @pre_check = options[:pre_check]
       @index = index
-      @queue_connection_params = options[:queue_connection_params]
+      @name = "W#{@index}"
       @setup_worker = options[:setup_worker]
       @teardown_worker = options[:teardown_worker]
       @worker_delay = options[:worker_delay]
@@ -43,6 +25,19 @@ module ParallelCucumber
       @log_dir = options[:log_dir]
       @log_file = "#{@log_dir}/worker_#{index}.log"
       @stdout_logger = stdout_logger # .sync writes only.
+      @is_busy_running_test = false
+      @jobs_queue = Queue.new
+      @manager = manager
+    end
+
+    attr_reader :index
+
+    def assign_job(instruction)
+      @jobs_queue.enq(instruction)
+    end
+
+    def busy_running_test?
+      @is_busy_running_test && @current_thread.alive?
     end
 
     def autoshutting_file
@@ -68,6 +63,9 @@ module ParallelCucumber
     end
 
     def start(env)
+      @current_thread = Thread.current
+      @manager.inform_idle(@name)
+
       env = env.dup.merge!('WORKER_LOG' => @log_file)
 
       File.delete(@log_file) if File.exist?(@log_file)
@@ -96,33 +94,31 @@ module ParallelCucumber
         begin
           setup(env)
 
-          queue = ParallelCucumber::Helper::Queue.new(@queue_connection_params)
-          directed_queue = ParallelCucumber::Helper::Queue.new(@queue_connection_params, "_#{@index}")
-          queue_tracker = Tracker.new(queue)
-
           loop_mm, loop_ss = time_it do
             loop do
-              break if queue.empty? && directed_queue.empty?
-              batch = []
-              precmd = precheck(env)
-              if (m = precmd.match(/precmd:retry-after-(\d+)-seconds/))
-                sleep(1 + m[1].to_i)
-                next
+              job = @jobs_queue.pop(false)
+              case job.type
+              when Job::PRECHECK
+                precmd = precheck(env)
+                if (m = precmd.match(/precmd:retry-after-(\d+)-seconds/))
+                  @manager.inform_idle(@name)
+                  sleep(1 + m[1].to_i)
+                  next
+                end
+                @manager.inform_healthy(@name)
+              when Job::RUN_TESTS
+                run_batch(env, results, running_total, job.details)
+                @manager.inform_idle(@name)
+              when Job::DIE
+                break
+              else
+                raise("Invalid job #{job.inspect}")
               end
-              @batch_size.times do
-                # TODO: Handle recovery of possibly toxic dequeued undirected tests if a worker dies mid-processing.
-                batch << (directed_queue.empty? ? queue : directed_queue).dequeue
-              end
-              batch.compact!
-              batch.sort! # Workaround for https://github.com/cucumber/cucumber-ruby/issues/952
-              break if batch.empty?
-
-              run_batch(env, queue_tracker, results, running_total, batch)
             end
           end
           @logger.debug("Loop took #{loop_mm} minutes #{loop_ss} seconds")
           @logger.update_into(@stdout_logger)
-        rescue => e
+        rescue StandardError => e
           trace = e.backtrace.join("\n\t").sub("\n\t", ": #{$ERROR_INFO}#{e.class ? " (#{e.class})" : ''}\n\t")
           @logger.error("Threw: #{e.inspect} #{trace}")
         ensure
@@ -135,10 +131,10 @@ module ParallelCucumber
       results
     end
 
-    def run_batch(env, queue_tracker, results, running_total, tests)
+    def run_batch(env, results, running_total, tests)
+      @is_busy_running_test = true
       batch_id = "#{Time.now.to_i}-#{@index}"
       @logger.debug("Batch ID is #{batch_id}")
-      @logger.info("Took #{tests.count} from the queue (#{queue_tracker.status}): #{tests.join(' ')}")
 
       batch_mm, batch_ss = time_it do
         begin
@@ -158,6 +154,7 @@ module ParallelCucumber
         process_results(batch_results, tests)
         running_totals(batch_results, running_total)
         results.merge!(batch_results)
+        @is_busy_running_test = false
       end
     ensure
       @logger.debug("Batch #{batch_id} took #{batch_mm} minutes #{batch_ss} seconds")

data/lib/parallel_cucumber/worker_manager.rb

@@ -0,0 +1,152 @@
+module ParallelCucumber
+  class WorkerManager
+    def initialize(options, logger)
+      @options = options
+      @batch_size = options[:batch_size]
+      @logger = logger
+      @queue_connection_params = options[:queue_connection_params]
+      @backlog = ParallelCucumber::Helper::Queue.new(@queue_connection_params)
+      @queue_tracker = Tracker.new(@backlog)
+      @back_up_worker_size = options[:backup_worker_count]
+      @directed_queues = Hash.new do |hash, key|
+        hash[key] = ParallelCucumber::Helper::Queue.new(@queue_connection_params, "_#{key}")
+      end
+      @workers = {}
+      @unchecked_workers = Queue.new
+      @healthy_workers = Queue.new
+    end
+
+    def start(number_of_workers)
+      create_workers(number_of_workers)
+      start_managing
+      start_workers
+    end
+
+    def kill
+      @current_thread.kill
+    end
+
+    def inform_healthy(worker)
+      @healthy_workers.enq(worker)
+    end
+
+    def inform_idle(worker)
+      @unchecked_workers.enq(worker)
+    end
+
+    private
+
+    def create_workers(number_of_workers)
+      number_of_workers.times do |index|
+        @workers["W#{index}"] =
+            ParallelCucumber::Worker.new(options: @options, index: index, stdout_logger: @logger, manager: self)
+      end
+    end
+
+    def start_managing
+      @current_thread = Thread.start do
+        loop do
+          if !@backlog.empty?
+            pre_check_unchecked_workers
+            give_job_to_healthy_worker
+          elsif any_worker_busy?
+            kill_surplus_workers
+            sleep 0.5
+          else
+            kill_all_workers
+            break
+          end
+        end
+      end
+    end
+
+    def start_workers
+      indices = (0...@workers.size).to_a
+      @results = Parallel.map(indices.dup, in_threads: @workers.size,
+                              finish: ->(_, ix, _) { @logger.synch { |l| l.info("Finished: #{ix} remaining: #{indices -= [ix]}") } }) do |index|
+        puts "Starting W#{index}"
+        @workers["W#{index}"].start(env_for_worker(@options[:env_variables], index))
+      end
+      @results.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
+    end
+
+    def kill_all_workers
+      @workers.values.each { |w| w.assign_job(Job.new(Job::DIE)) }
+    end
+
+    def kill_surplus_workers
+      until (@unchecked_workers.size + @healthy_workers.size) <= @back_up_worker_size
+        queue = !@unchecked_workers.empty? ? @unchecked_workers : @healthy_workers
+        worker = queue.pop(true)
+        @logger.info("Backup workers more than #{@back_up_worker_size}, killing #{worker}")
+        @workers[worker].assign_job(Job.new(Job::DIE))
+      end
+    end
+
+    def pre_check_unchecked_workers
+      while !@unchecked_workers.empty? && worker = @unchecked_workers.pop(false)
+        @logger.info("=== #{worker} was asked precheck")
+        @workers[worker].assign_job(Job.new(Job::PRECHECK))
+      end
+    end
+
+    def give_job_to_healthy_worker
+      return if @healthy_workers.empty?
+
+      worker_name = @healthy_workers.pop(true)
+      worker = @workers[worker_name]
+      batch = []
+      directed_queue = @directed_queues[worker.index]
+      @batch_size.times do
+        batch << (directed_queue.empty? ? @backlog : directed_queue).dequeue
+      end
+      batch.compact!
+      @logger.info("=== #{worker_name} was assigned #{batch.count} from the queue (#{@queue_tracker.status}): #{batch.join(' ')}")
+      worker.assign_job(Job.new(Job::RUN_TESTS, batch))
+    end
+
+    def any_worker_busy?
+      @workers.values.any?(&:busy_running_test?)
+    end
+
+    def env_for_worker(env_variables, worker_number)
+      env = env_variables.map do |k, v|
+        case v
+        when String, Numeric, TrueClass, FalseClass
+          [k, v]
+        when Array
+          [k, v[worker_number]]
+        when Hash
+          value = v[worker_number.to_s]
+          [k, value] unless value.nil?
+        when NilClass
+        else
+          raise("Don't know how to set '#{v}'<#{v.class}> to the environment variable '#{k}'")
+        end
+      end.compact.to_h
+
+      # Defaults, if absent in env. Shame 'merge' isn't something non-commutative like 'adopts/defaults'.
+      env = { TEST: 1, TEST_PROCESS_NUMBER: worker_number, WORKER_INDEX: worker_number }.merge(env)
+
+      # Overwrite this if it exists in env.
+      env.merge(PARALLEL_CUCUMBER_EXPORTS: env.keys.join(',')).map { |k, v| [k.to_s, v.to_s] }.to_h
+    end
+  end
+
+  class Tracker
+    def initialize(queue)
+      @backlog = queue
+    end
+
+    def status
+      queue_length = @backlog.length
+      now = Time.now
+      @full ||= queue_length
+      @start ||= now
+      completed = @full - queue_length
+      elapsed = now - @start
+      estimate = (completed == 0) ? '' : " #{(elapsed * @full / completed).to_i}s est"
+      "#{queue_length}/#{@full} left #{elapsed.to_i}s worker#{estimate}"
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallel_cucumber
 version: !ruby/object:Gem::Version
-  version: 0.2.16
+  version: 0.2.17
 platform: ruby
 authors:
 - Alexander Bayandin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-19 00:00:00.000000000 Z
+date: 2019-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cucumber
@@ -101,11 +101,13 @@ files:
 - lib/parallel_cucumber/helper/unittest/cucumber_test.rb
 - lib/parallel_cucumber/helper/utils.rb
 - lib/parallel_cucumber/hooks.rb
+- lib/parallel_cucumber/job.rb
 - lib/parallel_cucumber/logger.rb
 - lib/parallel_cucumber/main.rb
 - lib/parallel_cucumber/status.rb
 - lib/parallel_cucumber/version.rb
 - lib/parallel_cucumber/worker.rb
+- lib/parallel_cucumber/worker_manager.rb
 homepage: https://github.com/badoo/parallel_cucumber
 licenses:
 - MIT
@@ -125,8 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
 summary: Run cucumber in parallel