parallel_cucumber 0.2.0.pre.36 → 0.2.3

data/lib/parallel_cucumber/helper/cucumber.rb

@@ -9,27 +9,10 @@ module ParallelCucumber
   module Helper
     module Cucumber
       class << self
-        def dry_run_report(options, args)
-          options = options.dup
-          options = expand_profiles(options) unless config_file.nil?
-          options = remove_formatters(options)
-          content = nil
-
-          Tempfile.open(%w(dry-run .json)) do |f|
-            dry_run_options = "--dry-run --format json --out #{f.path}"
-
-            cmd = "cucumber #{options} #{dry_run_options} #{args.join(' ')}"
-            _stdout, stderr, status = Open3.capture3(cmd)
-            f.close
-
-            unless status == 0
-              cmd = "bundle exec #{cmd}" if ENV['BUNDLE_BIN_PATH']
-              raise("Can't generate dry run report: #{status}:\n\t#{cmd}\n\t#{stderr}")
-            end
-
-            content = File.read(f.path)
-          end
-          content
+        def selected_tests(options, args_string)
+          puts "selected_tests (#{options.inspect} #{args_string.inspect})"
+          dry_run_report = dry_run_report(options, args_string)
+          parse_json_report(dry_run_report).keys
         end
 
         def batch_mapped_files(options, batch, env)
@@ -52,7 +35,7 @@ module ParallelCucumber
                 next
               end
               steps = [background['steps'], scenario['steps']].flatten.compact
-              status = case
+              status = case # rubocop:disable Style/EmptyCaseCondition
                        when steps.map { |step| step['result'] }.all? { |result| result['status'] == 'skipped' }
                          Status::SKIPPED
                        when steps.map { |step| step['result'] }.any? { |result| result['status'] == 'failed' }
@@ -71,6 +54,29 @@ module ParallelCucumber
 
         private
 
+        def dry_run_report(options, args_string)
+          options = options.dup
+          options = expand_profiles(options) unless config_file.nil?
+          options = remove_formatters(options)
+          content = nil
+
+          Tempfile.open(%w(dry-run .json)) do |f|
+            dry_run_options = "--dry-run --format json --out #{f.path}"
+
+            cmd = "cucumber #{options} #{dry_run_options} #{args_string}"
+            _stdout, stderr, status = Open3.capture3(cmd)
+            f.close
+
+            unless status == 0
+              cmd = "bundle exec #{cmd}" if ENV['BUNDLE_BIN_PATH']
+              raise("Can't generate dry run report: #{status}:\n\t#{cmd}\n\t#{stderr}")
+            end
+
+            content = File.read(f.path)
+          end
+          content
+        end
+
         def expand_profiles(options, env = {})
           e = ENV.to_h
           ENV.replace(e.merge(env))

data/lib/parallel_cucumber/helper/processes.rb

@@ -2,39 +2,75 @@ module ParallelCucumber
   module Helper
     module Processes
       class << self
+        def ms_windows?
+          RUBY_PLATFORM =~ /mswin|mingw|migw32|cygwin/
+        end
+
+        def cp_rv(source, dest, logger = nil)
+          cp_out = if ms_windows?
+                     %x(powershell cp #{source} #{dest} -recurse -force 2>&1)
+                   else
+                     # Use system cp -r because Ruby's has crap diagnostics in weird situations.
+                     %x(cp -Rv #{source} #{dest} 2>&1)
+                   end
+          logger.debug("Copy of #{source} to #{dest} said: #{cp_out}") if logger
+        end
+
         def ps_tree
-          ` ps -ax -o ppid= -o pid= -o lstart= -o command= `
-            .each_line.map { |l| l.strip.split(/ +/, 3) }.to_a
-            .each_with_object({}) do |(ppid, pid, signature), tree|
-            (tree[pid] ||= { children: [] })[:signature] = signature
-            (tree[ppid] ||= { children: [] })[:children] << pid
+          if ms_windows?
+            system('powershell scripts/process_tree.ps1')
+          else
+            %x(ps -ax -o ppid= -o pid= -o lstart= -o command=)
+              .each_line.map { |l| l.strip.split(/ +/, 3) }.to_a
+              .each_with_object({}) do |(ppid, pid, signature), tree|
+                (tree[pid] ||= { children: [] })[:signature] = signature
+                (tree[ppid] ||= { children: [] })[:children] << pid
+              end
           end
         end
 
-        def kill_tree(sig, root, tree = nil, old_tree = nil)
-          descendants(root, tree, old_tree) do |pid|
-            begin
-              Process.kill(sig, pid.to_i)
-            rescue Errno::ESRCH
-              nil # It's gone already? Hurrah!
+        def kill_tree(sig, root, logger, tree = nil, old_tree = nil)
+          if ms_windows?
+            system("taskkill /pid #{root} /T")
+          else
+            descendants(root, logger, tree, old_tree, 'kill') do |pid, node|
+              begin
+                logger.warn "Killing #{node}"
+                Process.kill(sig, pid.to_i)
+              rescue Errno::ESRCH
+                nil # It's gone already? Hurrah!
+              end
             end
           end
+          # Let's kill pid unconditionally: descendants will go astray once reparented.
+          begin
+            logger.warn "Killing #{root} just in case"
+            Process.kill(sig, root.to_i)
+          rescue Errno::ESRCH
+            nil # It's gone already? Hurrah!
+          end
         end
 
-        def all_pids_dead?(root, tree = nil, old_tree = nil)
-          # Note: returns from THIS function as well as descendants: short-circuit evaluation.
-          descendants(root, tree, old_tree) { return false }
+        def all_pids_dead?(root, logger, tree = nil, old_tree = nil)
+          # Note: returns from THIS function as well as descendants: short-circuit evaluation if any descendants remain.
+          descendants(root, logger, tree, old_tree, 'dead?') { return false }
           true
         end
 
         # Walks old_tree, and yields all processes (alive or dead) that match the pid, start time, and command in
         # the new tree. Note that this will fumble children created since old_tree was created, but this thing is
         # riddled with race conditions anyway.
-        def descendants(pid, tree = nil, old_tree = nil, &block)
+        def descendants(pid, logger, tree = nil, old_tree = nil, why = '-', # rubocop:disable Metrics/ParameterLists
+                        level = 0, &block)
           tree ||= ps_tree
           old_tree ||= tree
-          old_tree[pid][:children].each { |c| descendants(c, tree, old_tree, &block) }
-          yield(pid) if tree[pid] && (tree[pid][:signature] == old_tree[pid][:signature])
+          old_tree_node = old_tree[pid]
+          unless old_tree_node
+            logger.warn "== old tree node went missing - #{why} - skipping subtree level=#{level}: #{pid}"
+            return
+          end
+          old_tree_node.fetch(:children, []).each { |c| descendants(c, logger, tree, old_tree, why, level + 1, &block) }
+          yield(pid, old_tree_node) if tree[pid] && (tree[pid][:signature] == old_tree_node[:signature])
         end
       end
     end

data/lib/parallel_cucumber/helper/queue.rb

@@ -5,7 +5,7 @@ module ParallelCucumber
     class Queue
       attr_reader :name
 
-      def initialize(queue_connection_params)
+      def initialize(queue_connection_params, append = '')
         # queue_connection_params:
         #   `url--[name]`
         # url:
@@ -15,11 +15,11 @@ module ParallelCucumber
         #   queue name, default is `queue`
         url, name = queue_connection_params
         @redis = Redis.new(url: url)
-        @name = name
+        @name = name + append
       end
 
       def enqueue(elements)
-        @redis.lpush(@name, elements)
+        @redis.lpush(@name, elements) unless elements.empty?
       end
 
       def dequeue

data/lib/parallel_cucumber/hooks.rb

@@ -0,0 +1,18 @@
+module ParallelCucumber
+  class Hooks
+    @after_batch_hooks ||= []
+
+    class << self
+      def register_after_batch( proc)
+        raise(ArgumentError, 'Please provide a valid callback') unless proc.respond_to?(:call)
+        @after_batch_hooks << proc
+      end
+
+      def fire_after_batch_hooks(*args)
+        @after_batch_hooks.each do |hook|
+          hook.call(*args)
+        end
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/logger.rb

@@ -2,8 +2,46 @@ require 'logger'
 
 module ParallelCucumber
   class CustomLogger < Logger
+    def initialize(*)
+      super
+      @mark = 0
+      # Don't want to log half-lines.
+      @incomplete_line = nil
+    end
+
+    def synch
+      mutex.synchronize { yield self }
+    end
+
+    def update_into(other_logger)
+      # TODO: This should write the #teamcity block wrapper: update(other_logger, 'qa-w12> precheck') etc.
+      @logdev.dev.fsync # Helpful, but inadequate: a child process might still have buffered stuff.
+      other_logger.synch do |l|
+        l << File.open(@logdev.filename || @logdev.dev.path) do |f|
+          begin
+            f.seek(@mark)
+            lines = f.readlines
+            if @incomplete_line && lines.count > 0
+              lines[0] = @incomplete_line + lines[0]
+              @incomplete_line = nil
+            end
+            unless lines.last && lines.last.end_with?("\n", "\r")
+              @incomplete_line = lines.pop
+            end
+            lines.join
+          ensure
+            @mark = f.tell
+          end
+        end
+      end
+    end
+
     private
 
+    def mutex
+      @mutex ||= Mutex.new
+    end
+
     def format_message(severity, datetime, progname, msg)
       if @level == DEBUG
         "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}]\t#{progname}\t#{severity}\t#{msg.gsub(/\s+/, ' ').strip}\n"

data/lib/parallel_cucumber/main.rb

@@ -8,93 +8,90 @@ module ParallelCucumber
       @options = options
 
       @logger = ParallelCucumber::CustomLogger.new(STDOUT)
+      load_external_files
       @logger.progname = 'Primary' # Longer than 'Main', to make the log file pretty.
       @logger.level = options[:debug] ? ParallelCucumber::CustomLogger::DEBUG : ParallelCucumber::CustomLogger::INFO
     end
 
+    def load_external_files
+      return if @options[:load_files].nil?
+      @options[:load_files].each do |file|
+        @logger.debug("Loading File: #{file}")
+        load file
+      end
+    end
+
     def run
-      queue = Helper::Queue.new(@options[:queue_connection_params])
       @logger.debug("Connecting to Queue: #{@options[:queue_connection_params]}")
+      queue = Helper::Queue.new(@options[:queue_connection_params])
 
       unless queue.empty?
         @logger.error("Queue '#{queue.name}' is not empty")
         exit(1)
       end
 
-      tests = []
-      mm, ss = time_it do
-        dry_run_report = Helper::Cucumber.dry_run_report(@options[:cucumber_options], @options[:cucumber_args])
-        tests = Helper::Cucumber.parse_json_report(dry_run_report).keys
-      end
-      tests.shuffle!
-      @logger.debug("Generating all tests took #{mm} minutes #{ss} seconds")
+      all_tests = Helper::Cucumber.selected_tests(@options[:cucumber_options], @options[:cucumber_args])
 
-      if tests.empty?
+      if all_tests.empty?
         @logger.error('There are no tests to run')
         exit(1)
       end
 
-      @logger.info("Adding #{tests.count} tests to Queue")
-      queue.enqueue(tests)
+      count = all_tests.count
 
-      if @options[:n] == 0
-        @options[:n] = [1, @options[:env_variables].map { |_k, v| v.is_a?(Array) ? v.count : 0 }].flatten.max
-        @logger.info("Inferred worker count #{@options[:n]} from env_variables option")
+      long_running_tests = if @options[:long_running_tests]
+                             Helper::Cucumber.selected_tests(@options[:cucumber_options], @options[:long_running_tests])
+                           else
+                             []
+                           end
+      first_tests = long_running_tests & all_tests
+      if !long_running_tests.empty? && first_tests.empty?
+        @logger.info("No long running tests found in common with main options: #{long_running_tests}")
       end
+      tests = first_tests + (all_tests - first_tests).shuffle
 
-      number_of_workers = [@options[:n], tests.count].min
-      unless number_of_workers == @options[:n]
-        @logger.info(<<-LOG)
-          Number of workers was overridden to #{number_of_workers}.
-          Was requested more workers (#{@options[:n]}) than tests (#{tests.count})".
-        LOG
+      @options[:directed_tests].each do |k, v|
+        directed_tests = Helper::Cucumber.selected_tests(@options[:cucumber_options], v)
+        if directed_tests.empty?
+          @logger.warn("Queue for #{k} is empty - nothing selected by #{v}")
+        else
+          directed_tests = (directed_tests & long_running_tests) + (directed_tests - long_running_tests).shuffle
+          @logger.debug("Connecting to Queue: _#{k}")
+          directed_queue = Helper::Queue.new(@options[:queue_connection_params], "_#{k}")
+          @logger.info("Adding #{directed_tests.count} tests to queue _#{k}")
+          directed_queue.enqueue(directed_tests)
+          tests -= directed_tests
+        end
       end
 
-      if (@options[:batch_size] - 1) * number_of_workers >= tests.count
-        original_batch_size = @options[:batch_size]
-        @options[:batch_size] = [(tests.count.to_f / number_of_workers).floor, 1].max
-        @logger.info(<<-LOG)
-          Batch size was overridden to #{@options[:batch_size]}.
-          Presumably it will be more optimal for #{tests.count} tests and #{number_of_workers} workers
-          than #{original_batch_size}
-        LOG
-      end
+      @logger.info("Adding #{tests.count} tests to Queue")
+      queue.enqueue(tests) unless tests.empty?
+
+      number_of_workers = determine_work_and_batch_size(count)
 
-      diff = []
-      info = {}
+      unrun = []
+      status_totals = {}
       total_mm, total_ss = time_it do
-        results = Helper::Command.wrap_block(@options[:log_decoration],
-                                             @options[:log_decoration]['worker_block'] || 'workers',
-                                             @logger) do
-          finished = []
-          Parallel.map(
-            0...number_of_workers,
-            in_processes: number_of_workers,
-            finish: -> (_, index, _) { @logger.info("Finished: #{finished[index] = index} #{finished - [nil]}") }
-          ) do |index|
-            Worker.new(@options, index).start(env_for_worker(@options[:env_variables], index))
-          end.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
-        end
-        results ||= {}
+        results = run_parallel_workers(number_of_workers) || {}
         unrun = tests - results.keys
-        @logger.error("Tests #{unrun.join(' ')} were not run") unless diff.empty?
+        @logger.error("Tests #{unrun.join(' ')} were not run") unless unrun.empty?
         @logger.error("Queue #{queue.name} is not empty") unless queue.empty?
 
-        Helper::Command.wrap_block(
-          @options[:log_decoration],
-          'Worker summary',
-          @logger
-        ) { results.find_all { |w| @logger.info("#{w.first} #{w.last.sort}") if w.first =~ /^:worker-/ } }
-
-        info = Status.constants.map do |status|
+        status_totals = Status.constants.map do |status|
           status = Status.const_get(status)
           tests_with_status = results.select { |_t, s| s == status }.keys
           [status, tests_with_status]
         end.to_h
+
+        Helper::Command.wrap_block(@options[:log_decoration], 'Worker summary', @logger) do
+          results.find_all { |w| w.first =~ /^:worker-/ }.each { |w| @logger.info("#{w.first} #{w.last.sort}") }
+        end
+
+        report_by_group(results)
       end
 
       @logger.debug("SUMMARY=#{@options[:summary]}") if @options[:summary]
-      info.each do |s, tt|
+      status_totals.each do |s, tt|
         next if tt.empty?
         @logger.info("Total: #{s.to_s.upcase} tests (#{tt.count}): #{tt.join(' ')}")
         filename = @options[:summary] && @options[:summary][s.to_s.downcase]
@@ -103,7 +100,78 @@ module ParallelCucumber
 
       @logger.info("\nTook #{total_mm} minutes #{total_ss} seconds")
 
-      exit((diff + info[Status::FAILED] + info[Status::UNKNOWN]).empty? ? 0 : 1)
+      exit((unrun + status_totals[Status::FAILED] + status_totals[Status::UNKNOWN]).empty? ? 0 : 1)
+    end
+
+    def report_by_group(results)
+      group = Hash.new { |h, k| h[k] = Hash.new(0) } # Default new keys to 0
+
+      Helper::Command.wrap_block(@options[:log_decoration], 'Worker summary', @logger) do
+        results.find_all { |w| w.first =~ /^:worker-/ }.each do |w|
+          # w = [:worker-0, [[:batches, 7], [:group, "localhost2"], [:skipped, 7]]]
+          gp = w.last[:group]
+          next unless gp
+          w.last.each { |(k, v)| group[gp][k] += w.last[k] if v && k != :group }
+          group[gp][:group] = {} unless group[gp].key?(:group)
+          group[gp][:group][w.first] = 1
+        end
+      end
+
+      @logger.info "== Groups key count #{group.keys.count}"
+
+      return unless group.keys.count > 1
+
+      Helper::Command.wrap_block(@options[:log_decoration], 'Group summary', @logger) do
+        group.each { |(k, v)| @logger.info("#{k} #{v.sort}") }
+      end
+    end
+
+    def run_parallel_workers(number_of_workers)
+      Helper::Command.wrap_block(@options[:log_decoration],
+                                 @options[:log_decoration]['worker_block'] || 'workers',
+                                 @logger) do
+        remaining = (0...number_of_workers).to_a
+        map = Parallel.map(
+          remaining.dup,
+          in_threads: number_of_workers,
+          finish: -> (_, ix, _) { @logger.synch { |l| l.info("Finished: #{ix} remaining: #{remaining -= [ix]}") } }
+        ) do |index|
+          ParallelCucumber::Worker
+            .new(@options, index, @logger)
+            .start(env_for_worker(@options[:env_variables], index))
+        end
+        map.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
+      end
+    end
+
+    def determine_work_and_batch_size(count)
+      if @options[:n] == 0
+        @options[:n] = [1, @options[:env_variables].map { |_k, v| v.is_a?(Array) ? v.count : 0 }].flatten.max
+        @logger.info("Inferred worker count #{@options[:n]} from env_variables option")
+      end
+
+      number_of_workers = [@options[:n], count].min
+      unless number_of_workers == @options[:n]
+        @logger.info(<<-LOG)
+          Number of workers was overridden to #{number_of_workers}.
+          More workers (#{@options[:n]}) requested than tests (#{count})".
+        LOG
+      end
+
+      @logger.info(<<-LOG)
+        Number of workers is #{number_of_workers}.
+      LOG
+
+      if (@options[:batch_size] - 1) * number_of_workers >= count
+        original_batch_size = @options[:batch_size]
+        @options[:batch_size] = [(count.to_f / number_of_workers).floor, 1].max
+        @logger.info(<<-LOG)
+          Batch size was overridden to #{@options[:batch_size]}.
+          Presumably it will be more optimal for #{count} tests and #{number_of_workers} workers
+          than #{original_batch_size}
+        LOG
+      end
+      number_of_workers
     end
 
     private