test-queue-split 0.3.0

data/lib/test_queue/runner.rb

@@ -0,0 +1,395 @@
+require 'socket'
+require 'fileutils'
+require 'securerandom'
+
+module TestQueue
+  class Worker
+    attr_accessor :pid, :status, :output, :stats, :num, :host
+    attr_accessor :start_time, :end_time
+    attr_accessor :summary, :failure_output
+
+    def initialize(pid, num)
+      @pid = pid
+      @num = num
+      @start_time = Time.now
+      @output = ''
+      @stats = {}
+    end
+
+    def lines
+      @output.split("\n")
+    end
+  end
+
+  class Runner
+    attr_accessor :concurrency
+
+    def initialize(queue, concurrency=nil, socket=nil, relay=nil)
+      raise ArgumentError, 'array required' unless Array === queue
+
+      if forced = ENV['TEST_QUEUE_FORCE']
+        forced = forced.split(/\s*,\s*/)
+        whitelist = Set.new(forced)
+        queue = queue.select{ |s| whitelist.include?(s.to_s) }
+        queue.sort_by!{ |s| forced.index(s.to_s) }
+      end
+
+      @procline = $0
+      @queue = queue
+      @suites = queue.inject(Hash.new){ |hash, suite| hash.update suite.to_s => suite }
+
+      @workers = {}
+      @completed = []
+
+      @concurrency =
+        concurrency ||
+        (ENV['TEST_QUEUE_WORKERS'] && ENV['TEST_QUEUE_WORKERS'].to_i) ||
+        if File.exists?('/proc/cpuinfo')
+          File.read('/proc/cpuinfo').split("\n").grep(/processor/).size
+        elsif RUBY_PLATFORM =~ /darwin/
+          `/usr/sbin/sysctl -n hw.activecpu`.to_i
+        else
+          2
+        end
+
+      @slave_connection_timeout =
+        (ENV['TEST_QUEUE_RELAY_TIMEOUT'] && ENV['TEST_QUEUE_RELAY_TIMEOUT'].to_i) ||
+        30
+
+      @run_token = ENV['TEST_QUEUE_RELAY_TOKEN'] || SecureRandom.hex(8)
+
+      @socket =
+        socket ||
+        ENV['TEST_QUEUE_SOCKET'] ||
+        "/tmp/test_queue_#{$$}_#{object_id}.sock"
+
+      @relay =
+        relay ||
+        ENV['TEST_QUEUE_RELAY']
+
+      @slave_message = ENV["TEST_QUEUE_SLAVE_MESSAGE"] if ENV.has_key?("TEST_QUEUE_SLAVE_MESSAGE")
+
+      if @relay == @socket
+        STDERR.puts "*** Detected TEST_QUEUE_RELAY == TEST_QUEUE_SOCKET. Disabling relay mode."
+        @relay = nil
+      elsif @relay
+        @queue = []
+      end
+    end
+
+    def stats
+      @stats ||=
+        if File.exists?(file = stats_file)
+          Marshal.load(IO.binread(file)) || {}
+        else
+          {}
+        end
+    end
+
+    def execute
+      $stdout.sync = $stderr.sync = true
+      @start_time = Time.now
+
+      @concurrency > 0 ?
+        execute_parallel :
+        execute_sequential
+    ensure
+      summarize_internal unless $!
+    end
+
+    def summarize_internal
+      puts
+      puts "==> Summary (#{@completed.size} workers in %.4fs)" % (Time.now-@start_time)
+      puts
+
+      @failures = ''
+      @completed.each do |worker|
+        summarize_worker(worker)
+        @failures << worker.failure_output if worker.failure_output
+
+        puts "    [%2d] %60s      %4d suites in %.4fs      (pid %d exit %d%s)" % [
+          worker.num,
+          worker.summary,
+          worker.stats.size,
+          worker.end_time - worker.start_time,
+          worker.pid,
+          worker.status.exitstatus,
+          worker.host && " on #{worker.host.split('.').first}"
+        ]
+      end
+
+      unless @failures.empty?
+        puts
+        puts "==> Failures"
+        puts
+        puts @failures
+      end
+
+      puts
+
+      if @stats
+        File.open(stats_file, 'wb') do |f|
+          f.write Marshal.dump(stats)
+        end
+      end
+
+      summarize
+
+      estatus = @completed.inject(0){ |s, worker| s + worker.status.exitstatus }
+      estatus = 255 if estatus > 255
+      exit!(estatus)
+    end
+
+    def summarize
+    end
+
+    def stats_file
+      ENV['TEST_QUEUE_STATS'] ||
+      '.test_queue_stats'
+    end
+
+    def execute_sequential
+      exit! run_worker(@queue)
+    end
+
+    def execute_parallel
+      start_master
+      prepare(@concurrency)
+      @prepared_time = Time.now
+      start_relay if relay?
+      spawn_workers
+      distribute_queue
+    ensure
+      stop_master
+
+      @workers.each do |pid, worker|
+        Process.kill 'KILL', pid
+      end
+
+      until @workers.empty?
+        reap_worker
+      end
+    end
+
+    def start_master
+      if !relay?
+        if @socket =~ /^(?:(.+):)?(\d+)$/
+          address = $1 || '0.0.0.0'
+          port = $2.to_i
+          @socket = "#$1:#$2"
+          @server = TCPServer.new(address, port)
+        else
+          FileUtils.rm(@socket) if File.exists?(@socket)
+          @server = UNIXServer.new(@socket)
+        end
+      end
+
+      desc = "test-queue master (#{relay?? "relaying to #{@relay}" : @socket})"
+      puts "Starting #{desc}"
+      $0 = "#{desc} - #{@procline}"
+    end
+
+    def start_relay
+      return unless relay?
+
+      sock = connect_to_relay
+      message = @slave_message ? " #{@slave_message}" : ""
+      message.gsub!(/(\r|\n)/, "") # Our "protocol" is newline-separated
+      sock.puts("SLAVE #{@concurrency} #{Socket.gethostname} #{@run_token}#{message}")
+      response = sock.gets.strip
+      unless response == "OK"
+        STDERR.puts "*** Got non-OK response from master: #{response}"
+        sock.close
+        exit! 1
+      end
+      sock.close
+    rescue Errno::ECONNREFUSED
+      STDERR.puts "*** Unable to connect to relay #{@relay}. Aborting.."
+      exit! 1
+    end
+
+    def stop_master
+      return if relay?
+
+      FileUtils.rm_f(@socket) if @socket && @server.is_a?(UNIXServer)
+      @server.close rescue nil if @server
+      @socket = @server = nil
+    end
+
+    def spawn_workers
+      @concurrency.times do |i|
+        num = i+1
+
+        pid = fork do
+          @server.close if @server
+
+          iterator = Iterator.new(relay?? @relay : @socket, @suites, method(:around_filter))
+          after_fork_internal(num, iterator)
+          ret = run_worker(iterator) || 0
+          cleanup_worker
+          Kernel.exit! ret
+        end
+
+        @workers[pid] = Worker.new(pid, num)
+      end
+    end
+
+    def after_fork_internal(num, iterator)
+      srand
+
+      output = File.open("/tmp/test_queue_worker_#{$$}_output", 'w')
+
+      $stdout.reopen(output)
+      $stderr.reopen($stdout)
+      $stdout.sync = $stderr.sync = true
+
+      $0 = "test-queue worker [#{num}]"
+      puts
+      puts "==> Starting #$0 (#{Process.pid} on #{Socket.gethostname}) - iterating over #{iterator.sock}"
+      puts
+
+      after_fork(num)
+    end
+
+    # Run in the master before the fork. Used to create
+    # concurrency copies of any databases required by the
+    # test workers.
+    def prepare(concurrency)
+    end
+
+    def around_filter(suite)
+      yield
+    end
+
+    # Prepare a worker for executing jobs after a fork.
+    def after_fork(num)
+    end
+
+    # Entry point for internal runner implementations. The iterator will yield
+    # jobs from the shared queue on the master.
+    #
+    # Returns nothing. exits 0 on success.
+    # exits N on error, where N is the number of failures.
+    def run_worker(iterator)
+      iterator.each do |item|
+        puts "  #{item.inspect}"
+      end
+
+      return 0 # exit status
+    end
+
+    def cleanup_worker
+    end
+
+    def summarize_worker(worker)
+      worker.summary = ''
+      worker.failure_output = ''
+    end
+
+    def reap_worker(blocking=true)
+      if pid = Process.waitpid(-1, blocking ? 0 : Process::WNOHANG) and worker = @workers.delete(pid)
+        worker.status = $?
+        worker.end_time = Time.now
+
+        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_output")
+          worker.output = IO.binread(file)
+          FileUtils.rm(file)
+        end
+
+        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_stats")
+          worker.stats = Marshal.load(IO.binread(file))
+          FileUtils.rm(file)
+        end
+
+        relay_to_master(worker) if relay?
+        worker_completed(worker)
+      end
+    end
+
+    def worker_completed(worker)
+      @completed << worker
+      puts worker.output if ENV['TEST_QUEUE_VERBOSE'] || worker.status.exitstatus != 0
+    end
+
+    def distribute_queue
+      return if relay?
+      remote_workers = 0
+
+      until @queue.empty? && remote_workers == 0
+        if IO.select([@server], nil, nil, 0.1).nil?
+          reap_worker(false) if @workers.any? # check for worker deaths
+        else
+          sock = @server.accept
+          cmd = sock.gets.strip
+          case cmd
+          when /^POP/
+            # If we have a slave from a different test run, don't respond, and it will consider the test run done.
+            if obj = @queue.shift
+              data = Marshal.dump(obj.to_s)
+              sock.write(data)
+            end
+          when /^SLAVE (\d+) ([\w\.-]+) (\w+)(?: (.+))?/
+            num = $1.to_i
+            slave = $2
+            run_token = $3
+            slave_message = $4
+            if run_token == @run_token
+              # If we have a slave from a different test run, don't respond, and it will consider the test run done.
+              sock.write("OK\n")
+              remote_workers += num
+            else
+              STDERR.puts "*** Worker from run #{run_token} connected to master for run #{@run_token}; ignoring."
+              sock.write("WRONG RUN\n")
+            end
+            message = "*** #{num} workers connected from #{slave} after #{Time.now-@start_time}s"
+            message << " " + slave_message if slave_message
+            STDERR.puts message
+          when /^WORKER (\d+)/
+            data = sock.read($1.to_i)
+            worker = Marshal.load(data)
+            worker_completed(worker)
+            remote_workers -= 1
+          end
+          sock.close
+        end
+      end
+    ensure
+      stop_master
+
+      until @workers.empty?
+        reap_worker
+      end
+    end
+
+    def relay?
+      !!@relay
+    end
+
+    def connect_to_relay
+      sock = nil
+      start = Time.now
+      puts "Attempting to connect for #{@slave_connection_timeout}s..."
+      while sock.nil?
+        begin
+          sock = TCPSocket.new(*@relay.split(':'))
+        rescue Errno::ECONNREFUSED => e
+          raise e if Time.now - start > @slave_connection_timeout
+          puts "Master not yet available, sleeping..."
+          sleep 0.5
+        end
+      end
+      sock
+    end
+
+    def relay_to_master(worker)
+      worker.host = Socket.gethostname
+      data = Marshal.dump(worker)
+
+      sock = connect_to_relay
+      sock.puts("WORKER #{data.bytesize}")
+      sock.write(data)
+    ensure
+      sock.close if sock
+    end
+  end
+end

data/lib/test_queue/version.rb

@@ -0,0 +1,4 @@
+module TestQueue
+  VERSION = '0.3.0' unless defined? ::TestQueue::VERSION
+  DATE    = '2015-08-31' unless defined? ::TestQueue::DATE
+end

data/lib/test_queue.rb

@@ -0,0 +1,8 @@
+if !IO.respond_to?(:binread)
+  class << IO
+    alias :binread :read
+  end
+end
+
+require 'test_queue/iterator'
+require 'test_queue/runner'

data/test/sample_minispec.rb

@@ -0,0 +1,31 @@
+require 'minitest/spec'
+
+class Meme
+  def i_can_has_cheezburger?
+    "OHAI!"
+  end
+
+  def will_it_blend?
+    "YES!"
+  end
+end
+
+describe Meme do
+  before do
+    @meme = Meme.new
+  end
+
+  describe "when asked about cheeseburgers" do
+    it "must respond positively" do
+      sleep 0.1
+      @meme.i_can_has_cheezburger?.must_equal "OHAI!"
+    end
+  end
+
+  describe "when asked about blending possibilities" do
+    it "won't say no" do
+      sleep 0.1
+      @meme.will_it_blend?.wont_match /^no/i
+    end
+  end
+end

data/test/sample_minitest4.rb

@@ -0,0 +1,23 @@
+require 'minitest/unit'
+
+class MiniTestEqual < MiniTest::Unit::TestCase
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+30.times do |i|
+  Object.const_set("MiniTestSleep#{i}", Class.new(MiniTest::Unit::TestCase) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
+end
+
+class MiniTestFailure < MiniTest::Unit::TestCase
+  def test_fail
+    assert_equal 0, 1
+  end
+end

data/test/sample_minitest5.rb

@@ -0,0 +1,23 @@
+require 'minitest/autorun'
+
+class MiniTestEqual < MiniTest::Test
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+30.times do |i|
+  Object.const_set("MiniTestSleep#{i}", Class.new(MiniTest::Test) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
+end
+
+class MiniTestFailure < MiniTest::Test
+  def test_fail
+    assert_equal 0, 1
+  end
+end

data/test/sample_spec.rb

@@ -0,0 +1,23 @@
+require 'rspec'
+
+describe 'RSpecEqual' do
+  it 'checks equality' do
+    expect(1).to eq 1
+  end
+end
+
+30.times do |i|
+  describe "RSpecSleep(#{i})" do
+    it "sleeps" do
+      start = Time.now
+      sleep(0.25)
+      expect(Time.now-start).to be_within(0.02).of(0.25)
+    end
+  end
+end
+
+describe 'RSpecFailure' do
+  it 'fails' do
+    expect(:foo).to eq :bar
+  end
+end

data/test/sample_testunit.rb

@@ -0,0 +1,23 @@
+require 'test/unit'
+
+class TestUnitEqual < Test::Unit::TestCase
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+30.times do |i|
+  Object.const_set("TestUnitSleep#{i}", Class.new(Test::Unit::TestCase) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
+end
+
+class TestUnitFailure < Test::Unit::TestCase
+  def test_fail
+    assert_equal 0, 1
+  end
+end

data/test-multi.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -x
+
+# export TEST_QUEUE_VERBOSE=1
+TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec minitest-queue ./test/sample_minitest5.rb &
+sleep 0.1
+TEST_QUEUE_RELAY=0.0.0.0:12345  bundle exec minitest-queue ./test/sample_minitest5.rb
+wait

data/test-queue-split.gemspec

@@ -0,0 +1,28 @@
+require_relative 'lib/test_queue/version'
+
+spec = Gem::Specification.new do |s|
+  s.name        = 'test-queue-split'
+  s.version     = TestQueue::VERSION
+  s.date        = TestQueue::DATE
+  s.summary     = 'parallel test runner'
+  s.description = 'minitest/rspec parallel test runner for CI environments'
+
+  s.homepage = 'http://github.com/tmm1/test-queue'
+
+  s.authors = ['Aman Gupta']
+  s.email   = 'ruby@tmm1.net'
+  s.license = 'MIT'
+
+  s.has_rdoc = false
+  s.bindir   = 'bin'
+  s.executables << 'rspec-queue'
+  s.executables << 'minitest-queue'
+  s.executables << 'testunit-queue'
+  s.executables << 'cucumber-queue'
+
+  s.add_development_dependency 'rspec', '>= 2.13', '< 4.0'
+  s.add_development_dependency 'cucumber', '~> 1.3.10'
+  s.add_development_dependency 'appium_thor', '~> 1.0.1'
+
+  s.files = `git ls-files`.split("\n")
+end

data/test.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -x
+
+export TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1
+
+export BUNDLE_GEMFILE=Gemfile-testunit
+bundle install
+bundle exec testunit-queue ./test/*_testunit.rb
+
+export BUNDLE_GEMFILE=Gemfile-minitest4
+bundle install
+bundle exec minitest-queue ./test/*_minitest4.rb
+bundle exec minitest-queue ./test/*_minispec.rb
+
+export BUNDLE_GEMFILE=Gemfile
+bundle install
+bundle exec minitest-queue ./test/*_minitest4.rb
+bundle exec minitest-queue ./test/*_minitest5.rb
+bundle exec minitest-queue ./test/*_minispec.rb
+bundle exec rspec-queue test
+bundle exec cucumber-queue
+
+TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,MiniTestSleep8,MiniTestFailure" bundle exec minitest-queue ./test/*_minitest5.rb