parallel_cucumber 0.1.22 → 0.2.0.pre.36

data/lib/parallel_cucumber/helper/cucumber.rb

@@ -0,0 +1,104 @@
+require 'English'
+require 'erb'
+require 'json'
+require 'open3'
+require 'tempfile'
+require 'yaml'
+
+module ParallelCucumber
+  module Helper
+    module Cucumber
+      class << self
+        def dry_run_report(options, args)
+          options = options.dup
+          options = expand_profiles(options) unless config_file.nil?
+          options = remove_formatters(options)
+          content = nil
+
+          Tempfile.open(%w(dry-run .json)) do |f|
+            dry_run_options = "--dry-run --format json --out #{f.path}"
+
+            cmd = "cucumber #{options} #{dry_run_options} #{args.join(' ')}"
+            _stdout, stderr, status = Open3.capture3(cmd)
+            f.close
+
+            unless status == 0
+              cmd = "bundle exec #{cmd}" if ENV['BUNDLE_BIN_PATH']
+              raise("Can't generate dry run report: #{status}:\n\t#{cmd}\n\t#{stderr}")
+            end
+
+            content = File.read(f.path)
+          end
+          content
+        end
+
+        def batch_mapped_files(options, batch, env)
+          options = options.dup
+          options = expand_profiles(options, env) unless config_file.nil?
+          file_map = {}
+          options.gsub!(/(?:\s|^)--dry-run\s+/, '')
+          options.gsub!(%r{((?:\s|^)(?:--out|-o))\s+((?:\S+\/)?(\S+))}) { "#{$1} #{file_map[$2] = "#{batch}/#{$3}"}" } # rubocop:disable Style/PerlBackrefs, Metrics/LineLength
+          [options, file_map]
+        end
+
+        def parse_json_report(json_report)
+          report = JSON.parse(json_report)
+          report.map do |feature|
+            next if feature['elements'].nil?
+            background = {}
+            feature['elements'].map do |scenario|
+              if scenario['type'] == 'background'
+                background = scenario
+                next
+              end
+              steps = [background['steps'], scenario['steps']].flatten.compact
+              status = case
+                       when steps.map { |step| step['result'] }.all? { |result| result['status'] == 'skipped' }
+                         Status::SKIPPED
+                       when steps.map { |step| step['result'] }.any? { |result| result['status'] == 'failed' }
+                         Status::FAILED
+                       when steps.map { |step| step['result'] }.all? { |result| result['status'] == 'passed' }
+                         Status::PASSED
+                       when steps.map { |step| step['result'] }.any? { |result| result['status'] == 'undefined' }
+                         Status::UNKNOWN
+                       else
+                         Status::UNKNOWN
+                       end
+              { "#{feature['uri']}:#{scenario['line']}".to_sym => status }
+            end
+          end.flatten.compact.inject(&:merge) || {}
+        end
+
+        private
+
+        def expand_profiles(options, env = {})
+          e = ENV.to_h
+          ENV.replace(e.merge(env))
+          begin
+            config = YAML.load(ERB.new(File.read(config_file)).result)
+            _expand_profiles(options, config)
+          ensure
+            ENV.replace(e)
+          end
+        end
+
+        def config_file
+          Dir.glob('{,.config/,config/}cucumber{.yml,.yaml}').first
+        end
+
+        def _expand_profiles(options, config)
+          profiles = options.scan(/(?:^|\s)((?:--profile|-p)\s+[\S]+)/)
+          profiles.map(&:first).each do |profile|
+            option = profile.gsub(/(--profile|-p)\s+/, '')
+            options.gsub!(profile, _expand_profiles(config.fetch(option), config))
+          end
+          options.strip
+        end
+
+        def remove_formatters(options)
+          options.gsub(/(^|\s)(--format|-f|--out|-o)\s+[\S]+/, '\\1').gsub(/(\s|^)--dry-run\s+/, '\\1')
+        end
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/helper/multi_delegator.rb

@@ -0,0 +1,24 @@
+# http://stackoverflow.com/questions/6407141/how-can-i-have-ruby-logger-log-output-to-stdout-as-well-as-file
+# answered Jun 20 '11 at 11:03 jonas054
+
+class MultiDelegator
+  def initialize(*targets)
+    @targets = targets
+  end
+
+  def self.delegate(*methods)
+    methods.each do |m|
+      define_method(m) do |*args|
+        @targets.map { |t| t.send(m, *args) }
+      end
+    end
+    self
+  end
+
+  class <<self
+    alias to new
+  end
+end
+
+# log_file = File.open("debug.log", "a")
+# log = Logger.new MultiDelegator.delegate(:write, :close).to(STDOUT, log_file)

data/lib/parallel_cucumber/helper/processes.rb

@@ -0,0 +1,42 @@
+module ParallelCucumber
+  module Helper
+    module Processes
+      class << self
+        def ps_tree
+          ` ps -ax -o ppid= -o pid= -o lstart= -o command= `
+            .each_line.map { |l| l.strip.split(/ +/, 3) }.to_a
+            .each_with_object({}) do |(ppid, pid, signature), tree|
+            (tree[pid] ||= { children: [] })[:signature] = signature
+            (tree[ppid] ||= { children: [] })[:children] << pid
+          end
+        end
+
+        def kill_tree(sig, root, tree = nil, old_tree = nil)
+          descendants(root, tree, old_tree) do |pid|
+            begin
+              Process.kill(sig, pid.to_i)
+            rescue Errno::ESRCH
+              nil # It's gone already? Hurrah!
+            end
+          end
+        end
+
+        def all_pids_dead?(root, tree = nil, old_tree = nil)
+          # Note: returns from THIS function as well as descendants: short-circuit evaluation.
+          descendants(root, tree, old_tree) { return false }
+          true
+        end
+
+        # Walks old_tree, and yields all processes (alive or dead) that match the pid, start time, and command in
+        # the new tree. Note that this will fumble children created since old_tree was created, but this thing is
+        # riddled with race conditions anyway.
+        def descendants(pid, tree = nil, old_tree = nil, &block)
+          tree ||= ps_tree
+          old_tree ||= tree
+          old_tree[pid][:children].each { |c| descendants(c, tree, old_tree, &block) }
+          yield(pid) if tree[pid] && (tree[pid][:signature] == old_tree[pid][:signature])
+        end
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/helper/queue.rb

@@ -0,0 +1,38 @@
+require 'redis'
+
+module ParallelCucumber
+  module Helper
+    class Queue
+      attr_reader :name
+
+      def initialize(queue_connection_params)
+        # queue_connection_params:
+        #   `url--[name]`
+        # url:
+        #   TCP connection: `redis://[password]@[hostname]:[port]/[db]` (password, port and database are optional),
+        #   unix socket connection: `unix://[path to Redis socket]`.
+        # name:
+        #   queue name, default is `queue`
+        url, name = queue_connection_params
+        @redis = Redis.new(url: url)
+        @name = name
+      end
+
+      def enqueue(elements)
+        @redis.lpush(@name, elements)
+      end
+
+      def dequeue
+        @redis.rpop(@name)
+      end
+
+      def length
+        @redis.llen(@name)
+      end
+
+      def empty?
+        length.zero?
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/helper/unittest/cucumber_test.rb

@@ -0,0 +1,10 @@
+require_relative '../cucumber'
+require 'test/unit'
+
+class CucumberTest < Test::Unit::TestCase
+  def test_argument_mapping
+    s, m = ParallelCucumber::Helper::Cucumber.batch_mapped_files('--out foo/bar -o wib/ble', 'ARGH', {})
+    assert_equal('--out ARGH/bar -o ARGH/ble', s)
+    assert_equal({ 'foo/bar' => 'ARGH/bar', 'wib/ble' => 'ARGH/ble' }, m)
+  end
+end

data/lib/parallel_cucumber/helper/utils.rb

@@ -0,0 +1,13 @@
+module ParallelCucumber
+  module Helper
+    module Utils
+      def time_it
+        t1 = Time.now
+        yield
+        t2 = Time.now
+        mm, ss = (t2 - t1).divmod(60)
+        [mm, ss.round(1)]
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/logger.rb

@@ -0,0 +1,15 @@
+require 'logger'
+
+module ParallelCucumber
+  class CustomLogger < Logger
+    private
+
+    def format_message(severity, datetime, progname, msg)
+      if @level == DEBUG
+        "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}]\t#{progname}\t#{severity}\t#{msg.gsub(/\s+/, ' ').strip}\n"
+      else
+        "#{progname}\t#{msg.gsub(/\s+/, ' ').strip}\n"
+      end
+    end
+  end
+end

data/lib/parallel_cucumber/main.rb

@@ -0,0 +1,134 @@
+require 'parallel'
+
+module ParallelCucumber
+  class Main
+    include ParallelCucumber::Helper::Utils
+
+    def initialize(options)
+      @options = options
+
+      @logger = ParallelCucumber::CustomLogger.new(STDOUT)
+      @logger.progname = 'Primary' # Longer than 'Main', to make the log file pretty.
+      @logger.level = options[:debug] ? ParallelCucumber::CustomLogger::DEBUG : ParallelCucumber::CustomLogger::INFO
+    end
+
+    def run
+      queue = Helper::Queue.new(@options[:queue_connection_params])
+      @logger.debug("Connecting to Queue: #{@options[:queue_connection_params]}")
+
+      unless queue.empty?
+        @logger.error("Queue '#{queue.name}' is not empty")
+        exit(1)
+      end
+
+      tests = []
+      mm, ss = time_it do
+        dry_run_report = Helper::Cucumber.dry_run_report(@options[:cucumber_options], @options[:cucumber_args])
+        tests = Helper::Cucumber.parse_json_report(dry_run_report).keys
+      end
+      tests.shuffle!
+      @logger.debug("Generating all tests took #{mm} minutes #{ss} seconds")
+
+      if tests.empty?
+        @logger.error('There are no tests to run')
+        exit(1)
+      end
+
+      @logger.info("Adding #{tests.count} tests to Queue")
+      queue.enqueue(tests)
+
+      if @options[:n] == 0
+        @options[:n] = [1, @options[:env_variables].map { |_k, v| v.is_a?(Array) ? v.count : 0 }].flatten.max
+        @logger.info("Inferred worker count #{@options[:n]} from env_variables option")
+      end
+
+      number_of_workers = [@options[:n], tests.count].min
+      unless number_of_workers == @options[:n]
+        @logger.info(<<-LOG)
+          Number of workers was overridden to #{number_of_workers}.
+          Was requested more workers (#{@options[:n]}) than tests (#{tests.count})".
+        LOG
+      end
+
+      if (@options[:batch_size] - 1) * number_of_workers >= tests.count
+        original_batch_size = @options[:batch_size]
+        @options[:batch_size] = [(tests.count.to_f / number_of_workers).floor, 1].max
+        @logger.info(<<-LOG)
+          Batch size was overridden to #{@options[:batch_size]}.
+          Presumably it will be more optimal for #{tests.count} tests and #{number_of_workers} workers
+          than #{original_batch_size}
+        LOG
+      end
+
+      diff = []
+      info = {}
+      total_mm, total_ss = time_it do
+        results = Helper::Command.wrap_block(@options[:log_decoration],
+                                             @options[:log_decoration]['worker_block'] || 'workers',
+                                             @logger) do
+          finished = []
+          Parallel.map(
+            0...number_of_workers,
+            in_processes: number_of_workers,
+            finish: -> (_, index, _) { @logger.info("Finished: #{finished[index] = index} #{finished - [nil]}") }
+          ) do |index|
+            Worker.new(@options, index).start(env_for_worker(@options[:env_variables], index))
+          end.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
+        end
+        results ||= {}
+        unrun = tests - results.keys
+        @logger.error("Tests #{unrun.join(' ')} were not run") unless diff.empty?
+        @logger.error("Queue #{queue.name} is not empty") unless queue.empty?
+
+        Helper::Command.wrap_block(
+          @options[:log_decoration],
+          'Worker summary',
+          @logger
+        ) { results.find_all { |w| @logger.info("#{w.first} #{w.last.sort}") if w.first =~ /^:worker-/ } }
+
+        info = Status.constants.map do |status|
+          status = Status.const_get(status)
+          tests_with_status = results.select { |_t, s| s == status }.keys
+          [status, tests_with_status]
+        end.to_h
+      end
+
+      @logger.debug("SUMMARY=#{@options[:summary]}") if @options[:summary]
+      info.each do |s, tt|
+        next if tt.empty?
+        @logger.info("Total: #{s.to_s.upcase} tests (#{tt.count}): #{tt.join(' ')}")
+        filename = @options[:summary] && @options[:summary][s.to_s.downcase]
+        open(filename, 'w') { |f| f << tt.join("\n") } if filename
+      end
+
+      @logger.info("\nTook #{total_mm} minutes #{total_ss} seconds")
+
+      exit((diff + info[Status::FAILED] + info[Status::UNKNOWN]).empty? ? 0 : 1)
+    end
+
+    private
+
+    def env_for_worker(env_variables, worker_number)
+      env = env_variables.map do |k, v|
+        case v
+        when String, Numeric, TrueClass, FalseClass
+          [k, v]
+        when Array
+          [k, v[worker_number]]
+        when Hash
+          value = v[worker_number.to_s]
+          [k, value] unless value.nil?
+        when NilClass
+        else
+          raise("Don't know how to set '#{v}'<#{v.class}> to the environment variable '#{k}'")
+        end
+      end.compact.to_h
+
+      # Defaults, if absent in env. Shame 'merge' isn't something non-commutative like 'adopts/defaults'.
+      env = { TEST: 1, TEST_PROCESS_NUMBER: worker_number, WORKER_INDEX: worker_number }.merge(env)
+
+      # Overwrite this if it exists in env.
+      env.merge(PARALLEL_CUCUMBER_EXPORTS: env.keys.join(',')).map { |k, v| [k.to_s, v.to_s] }.to_h
+    end
+  end
+end

data/lib/parallel_cucumber/status.rb

@@ -0,0 +1,8 @@
+module ParallelCucumber
+  module Status
+    PASSED = :passed
+    FAILED = :failed
+    SKIPPED = :skipped
+    UNKNOWN = :unknown
+  end
+end

data/lib/parallel_cucumber/version.rb

@@ -1,3 +1,3 @@
 module ParallelCucumber
-  VERSION = '0.1.22'.freeze
-end # ParallelCucumber
+  VERSION = '0.2.0.pre.36'.freeze
+end

data/lib/parallel_cucumber/worker.rb

@@ -0,0 +1,239 @@
+require 'English'
+require 'timeout'
+
+module ParallelCucumber
+  class Tracker
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def status
+      queue_length = @queue.length
+      now = Time.now
+      @full ||= queue_length
+      @start ||= now
+      completed = @full - queue_length
+      elapsed = now - @start
+      estimate = (completed == 0) ? '' : " #{(elapsed * @full / completed).to_i}s est"
+      "#{queue_length}/#{@full} left #{elapsed.to_i}s worker#{estimate}"
+    end
+  end
+
+  class Worker
+    include ParallelCucumber::Helper::Utils
+
+    def initialize(options, index)
+      @batch_size = options[:batch_size]
+      @batch_timeout = options[:batch_timeout]
+      @setup_timeout = options[:setup_timeout]
+      @cucumber_options = options[:cucumber_options]
+      @test_command = options[:test_command]
+      @pre_check = options[:pre_check]
+      @pretty = options[:pretty]
+      @env_variables = options[:env_variables]
+      @index = index
+      @queue_connection_params = options[:queue_connection_params]
+      @setup_worker = options[:setup_worker]
+      @teardown_worker = options[:teardown_worker]
+      @worker_delay = options[:worker_delay]
+      @debug = options[:debug]
+      @log_decoration = options[:log_decoration]
+      @log_dir = options[:log_dir]
+      @log_file = "#{@log_dir}/worker_#{index}.log"
+    end
+
+    def start(env)
+      env = env.dup.merge!('WORKER_LOG' => @log_file)
+
+      File.delete(@log_file) if File.exist?(@log_file)
+      File.open(@log_file, 'a') do |file_handle|
+        file_handle.sync = true
+        @logger = ParallelCucumber::CustomLogger.new(MultiDelegator.delegate(:write, :close).to(STDOUT, file_handle))
+        @logger.progname = "Worker-#{@index}"
+        @logger.level = @debug ? ParallelCucumber::CustomLogger::DEBUG : ParallelCucumber::CustomLogger::INFO
+
+        @logger.info("Starting, also logging to #{@log_file}")
+
+        unless @worker_delay.zero?
+          @logger.info("Waiting #{@worker_delay * @index} seconds before start")
+          sleep(@worker_delay * @index)
+        end
+
+        @logger.debug(<<-LOG)
+        Additional environment variables: #{env.map { |k, v| "#{k}=#{v}" }.join(' ')}
+        LOG
+
+        results = {}
+        running_total = Hash.new(0)
+        begin
+          setup(env)
+
+          queue = ParallelCucumber::Helper::Queue.new(@queue_connection_params)
+          queue_tracker = Tracker.new(queue)
+
+          loop_mm, loop_ss = time_it do
+            loop do
+              break if queue.empty?
+              batch = []
+              precheck(env)
+              @batch_size.times do
+                # TODO: Handle recovery of dequeued tests, if a worker dies mid-processing.
+                batch << queue.dequeue
+              end
+              batch.compact!
+              batch.sort!
+              break if batch.empty?
+
+              run_batch(env, queue_tracker, results, running_total, batch)
+            end
+          end
+          @logger.debug("Loop took #{loop_mm} minutes #{loop_ss} seconds")
+        ensure
+          teardown(env)
+
+          results[":worker-#{@index}"] = running_total
+          results
+        end
+      end
+    end
+
+    def run_batch(env, queue_tracker, results, running_total, tests)
+      batch_id = "#{Time.now.to_i}-#{@index}"
+      @logger.debug("Batch ID is #{batch_id}")
+      @logger.info("Took #{tests.count} from the queue (#{queue_tracker.status}): #{tests.join(' ')}")
+
+      batch_mm, batch_ss = time_it do
+        batch_results = test_batch(batch_id, env, running_total, tests)
+
+        process_results(batch_results, tests)
+
+        running_totals(batch_results, running_total)
+        results.merge!(batch_results)
+      end
+
+      @logger.debug("Batch #{batch_id} took #{batch_mm} minutes #{batch_ss} seconds")
+    end
+
+    def precheck(env)
+      return unless @pre_check
+      continue = Helper::Command.exec_command(
+        env, 'precheck', @pre_check, @log_file, @logger, @log_decoration, @batch_timeout
+      )
+      return if continue
+      @logger.error('Pre-check failed: quitting immediately')
+      raise :prechek_failed
+    end
+
+    def running_totals(batch_results, running_total)
+      batch_info = Status.constants.map do |status|
+        status = Status.const_get(status)
+        [status, batch_results.select { |_t, s| s == status }.keys]
+      end.to_h
+      batch_info.each do |s, tt|
+        @logger.info("#{s.to_s.upcase} #{tt.count} tests: #{tt.join(' ')}") unless tt.empty?
+        running_total[s] += tt.count unless tt.empty?
+      end
+      running_total[:batches] += 1
+      @logger.info(running_total.sort.to_s)
+    end
+
+    def process_results(batch_results, tests)
+      batch_keys = batch_results.keys
+      test_syms = tests.map(&:to_sym)
+      unrun = test_syms - batch_keys
+      surfeit = batch_keys - test_syms
+      unrun.each { |test| batch_results[test] = Status::UNKNOWN }
+      surfeit.each { |test| batch_results.delete(test) }
+      @logger.error("Did not run #{unrun.count}/#{tests.count}: #{unrun.join(' ')}") unless unrun.empty?
+      @logger.error("Extraneous runs (#{surfeit.count}): #{surfeit.join(' ')}") unless surfeit.empty?
+      return if surfeit.empty?
+      # Don't see how this can happen, but...
+      @logger.error("Tests/result mismatch: #{tests.count}!=#{batch_results.count}: #{tests}/#{batch_keys}")
+    end
+
+    def test_batch(batch_id, env, running_total, tests)
+      test_batch_dir = "/tmp/w-#{batch_id}"
+      FileUtils.rm_rf(test_batch_dir)
+      FileUtils.mkpath(test_batch_dir)
+
+      test_state = "#{test_batch_dir}/test_state.json"
+      cmd = "#{@test_command} #{@pretty} --format json --out #{test_state} #{@cucumber_options} "
+      batch_env = {
+        :TEST_BATCH_ID.to_s => batch_id,
+        :TEST_BATCH_DIR.to_s => test_batch_dir,
+        :BATCH_NUMBER.to_s => running_total[:batches].to_s
+      }.merge(env)
+      mapped_batch_cmd, file_map = Helper::Cucumber.batch_mapped_files(cmd, test_batch_dir, batch_env)
+      file_map.each { |_user, worker| FileUtils.mkpath(worker) if worker =~ %r{\/$} }
+      mapped_batch_cmd += ' ' + tests.join(' ')
+      res = ParallelCucumber::Helper::Command.exec_command(
+        batch_env, 'batch', mapped_batch_cmd, @log_file, @logger, @log_decoration, @batch_timeout
+      )
+      batch_results = if res.nil?
+                        {}
+                      else
+                        Helper::Command.wrap_block(@log_decoration, 'file copy', @logger) do
+                          # Use system cp -r because Ruby's has crap diagnostics in weird situations.
+                          # Copy files we might have renamed or moved
+                          file_map.each do |user, worker|
+                            unless worker == user
+                              cp_out = `cp -Rv #{worker} #{user} 2>&1`
+                              @logger.debug("Copy of #{worker} to #{user} said: #{cp_out}")
+                            end
+                          end
+                          # Copy everything else too, in case it's interesting.
+                          cp_out = `cp -Rv #{test_batch_dir}/*  #{@log_dir} 2>&1`
+                          @logger.debug("Copy of #{test_batch_dir}/* to #{@log_dir} said: #{cp_out}")
+                          parse_results(test_state)
+                        end
+                      end
+    ensure
+      FileUtils.rm_rf(test_batch_dir)
+      batch_results
+    end
+
+    def teardown(env)
+      return unless @teardown_worker
+      mm, ss = time_it do
+        @logger.info('Teardown running')
+        success = Helper::Command.exec_command(
+          env, 'teardown', @teardown_worker, @log_file, @logger, @log_decoration
+        )
+        @logger.warn('Teardown finished with error') unless success
+      end
+      @logger.debug("Teardown took #{mm} minutes #{ss} seconds")
+    end
+
+    def setup(env)
+      return unless @setup_worker
+      mm, ss = time_it do
+        @logger.info('Setup running')
+        success = Helper::Command.exec_command(
+          env, 'setup', @setup_worker, @log_file, @logger, @log_decoration, @setup_timeout
+        )
+        unless success
+          @logger.warn('Setup failed: quitting immediately')
+          raise :setup_failed
+        end
+      end
+      @logger.debug("Setup took #{mm} minutes #{ss} seconds")
+    end
+
+    def parse_results(f)
+      unless File.file?(f)
+        @logger.error("Results file does not exist: #{f}")
+        return {}
+      end
+      json_report = File.read(f)
+      if json_report.empty?
+        @logger.error("Results file is empty: #{f}")
+        return {}
+      end
+      Helper::Cucumber.parse_json_report(json_report)
+    rescue => e
+      trace = e.backtrace.join("\n\t").sub("\n\t", ": #{$ERROR_INFO}#{e.class ? " (#{e.class})" : ''}\n\t")
+      @logger.error("Threw: JSON parse of results caused #{trace}")
+      {}
+    end
+  end
+end