test-queue 0.1.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,26 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.4)
+    minitest (4.7.3)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest
+  rspec
+  test-queue!

data/README.md

@@ -0,0 +1,58 @@
+## test-queue
+
+Yet another parallel test runner, built using a centralized queue to ensure
+optimal distribution of tests between workers.
+
+Specifially optimized for CI environments: build statistics from each run
+are stored locally and used to sort the queue at the beginning of the
+next run.
+
+### usage
+
+```
+$ minitest-queue $(find test/ -name \*_test.rb)
+$ rspec-queue    $(find spec/ -name \*_spec.rb)
+```
+
+### design
+
+test-queue uses a simple master + pre-fork worker model. The master
+exposes a unix domain socket server which workers use to grab tests off
+the queue.
+
+```
+─┬─ 21232 minitest-queue master
+ ├─── 21571 minitest-queue worker [3] - AuthenticationTest
+ ├─── 21568 minitest-queue worker [2] - ApiTest
+ ├─── 21565 minitest-queue worker [1] - UsersControllerTest
+ └─── 21562 minitest-queue worker [0] - UserTest
+```
+
+### customization
+
+Since test-queue uses `fork(2)` to spawn off workers, you must ensure each worker
+runs in an isolated environment. Use the `after_fork` hook with a custom
+runner to reset any global state:
+
+``` ruby
+class CustomMiniTestRunner < TestQueue::Runner::MiniTest
+  def after_fork(num)
+    super
+
+    # use separate mysql database (we assume it exists and has the right schema already)
+    ActiveRecord::Base.configurations['test']['database'] << num.to_s
+    ActiveRecord::Base.establish_connection(:test)
+
+    # use separate redis database
+    $redis.client.db = num
+    $redis.client.reconnect
+  end
+end
+
+CustomMiniTestRunner.new.execute
+```
+
+### see also
+
+  * https://github.com/Shopify/rails_parallel
+  * https://github.com/grosser/parallel_tests

data/bin/minitest-queue

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/minitest'
+ARGV.each{ |f| require(f) }
+TestQueue::Runner::MiniTest.new.execute

data/bin/rspec-queue

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/rspec'
+TestQueue::Runner::RSpec.new.execute

data/lib/test-queue.rb

@@ -0,0 +1 @@
+require 'test_queue'

data/lib/test_queue.rb

@@ -0,0 +1,8 @@
+if !IO.respond_to?(:binread)
+  class << IO
+    alias :binread :read
+  end
+end
+
+require 'test_queue/iterator'
+require 'test_queue/runner'

data/lib/test_queue/iterator.rb

@@ -0,0 +1,44 @@
+module TestQueue
+  class Iterator
+    attr_reader :stats
+
+    def initialize(sock)
+      @sock = sock
+      @done = false
+      @stats = {}
+    end
+
+    def each
+      fail 'already used this iterator' if @done
+
+      while true
+        client = UNIXSocket.new(@sock)
+        r, w, e = IO.select([client], nil, [client], nil)
+        break if !e.empty?
+
+        if data = client.read(16384)
+          client.close
+          item = Marshal.load(data)
+
+          start = Time.now
+          yield item
+          @stats[item] = Time.now - start
+        else
+          break
+        end
+      end
+    rescue Errno::ENOENT, Errno::ECONNRESET, Errno::ECONNREFUSED
+    ensure
+      @done = true
+      File.open("/tmp/test_queue_worker_#{$$}_stats", "wb") do |f|
+        f.write Marshal.dump(@stats)
+      end
+    end
+
+    include Enumerable
+
+    def empty?
+      false
+    end
+  end
+end

data/lib/test_queue/runner.rb

@@ -0,0 +1,201 @@
+require 'socket'
+require 'fileutils'
+
+module TestQueue
+  class Worker
+    attr_accessor :pid, :status, :output, :stats, :num
+    attr_accessor :start_time, :end_time
+
+    def initialize(pid, num)
+      @pid = pid
+      @num = num
+      @start_time = Time.now
+      @output = ''
+    end
+
+    def lines
+      @output.split("\n")
+    end
+  end
+
+  class Runner
+    attr_accessor :concurrency
+
+    def initialize(queue, concurrency=nil)
+      raise ArgumentError, 'array required' unless Array === queue
+
+      @queue = queue
+
+      @workers = {}
+      @completed = []
+
+      @concurrency =
+        concurrency ||
+        (ENV['TEST_QUEUE_WORKERS'] && ENV['TEST_QUEUE_WORKERS'].to_i) ||
+        if File.exists?('/proc/cpuinfo')
+          File.read('/proc/cpuinfo').split("\n").grep(/processor/).size
+        elsif RUBY_PLATFORM =~ /darwin/
+          `/usr/sbin/sysctl -n hw.activecpu`.to_i
+        else
+          2
+        end
+    end
+
+    def stats
+      @stats ||=
+        if File.exists?('.test_queue_stats')
+          Marshal.load(IO.binread('.test_queue_stats'))
+        else
+          {}
+        end
+    end
+
+    def execute
+      @concurrency > 0 ?
+        execute_parallel :
+        execute_sequential
+    ensure
+      puts
+      puts "==> Summary"
+      puts
+
+      @failures = ''
+      @completed.each do |worker|
+        summary, failures = summarize_worker(worker)
+        @failures << failures if failures
+
+        puts "    [%d] %55s      in %.4fs      (pid %d exit %d)" % [
+          worker.num,
+          summary,
+          worker.end_time - worker.start_time,
+          worker.pid,
+          worker.status.exitstatus
+        ]
+      end
+
+      unless @failures.empty?
+        puts
+        puts "==> Failures"
+        puts
+        puts @failures
+      end
+
+      puts
+
+      File.open('.test_queue_stats', 'wb') do |f|
+        f.write Marshal.dump(@stats)
+      end
+
+      exit! @completed.inject(0){ |s, worker| s + worker.status.exitstatus }
+    end
+
+    def execute_sequential
+      exit! run_worker(@queue)
+    end
+
+    def execute_parallel
+      start_master
+      spawn_workers
+      distribute_queue
+    ensure
+      stop_master
+
+      @workers.each do |pid, worker|
+        Process.kill 'KILL', pid
+      end
+
+      until @workers.empty?
+        cleanup_worker
+      end
+    end
+
+    def start_master
+      @socket = "/tmp/test_queue_#{$$}_#{object_id}.sock"
+      FileUtils.rm(@socket) if File.exists?(@socket)
+      @server = UNIXServer.new(@socket)
+    end
+
+    def stop_master
+      FileUtils.rm_f(@socket) if @socket
+      @server.close rescue nil if @server
+      @socket = @server = nil
+    end
+
+    def spawn_workers
+      @concurrency.times do |i|
+        pid = fork do
+          @server.close
+          after_fork(i)
+          exit! run_worker(iterator = Iterator.new(@socket)) || 0
+        end
+
+        @workers[pid] = Worker.new(pid, i)
+      end
+    end
+
+    def after_fork(num)
+      srand
+
+      output = File.open("/tmp/test_queue_worker_#{$$}_output", 'w')
+      output.sync = true
+
+      $stdout.reopen(output)
+      $stderr.reopen($stdout)
+
+      $0 = "ruby test-queue worker [#{num}]"
+      puts
+      puts "==> Starting #$0 (#{Process.pid})"
+      puts
+    end
+
+    def run_worker(iterator)
+      iterator.each do |item|
+        puts "  #{item.inspect}"
+      end
+
+      return 0 # exit status
+    end
+
+    def summarize_worker(worker)
+      num_tests = ''
+      failures = ''
+
+      [ num_tests, failures ]
+    end
+
+    def cleanup_worker
+      if pid = Process.waitpid and worker = @workers.delete(pid)
+        @completed << worker
+        worker.status = $?
+        worker.end_time = Time.now
+
+        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_output")
+          worker.output = IO.binread(file)
+          puts worker.output
+          FileUtils.rm(file)
+        end
+
+        if File.exists?(file = "/tmp/test_queue_worker_#{pid}_stats")
+          worker.stats = Marshal.load(IO.binread(file))
+          FileUtils.rm(file)
+        end
+      end
+    end
+
+    def distribute_queue
+      until @queue.empty?
+        IO.select([@server], nil, nil, nil)
+
+        sock = @server.accept
+        sock.write(Marshal.dump(@queue.shift))
+        sock.close
+      end
+    ensure
+      stop_master
+
+      until @workers.empty?
+        cleanup_worker
+      end
+    end
+  end
+end

data/lib/test_queue/runner/minitest.rb

@@ -0,0 +1,69 @@
+require 'test_queue/runner'
+require 'minitest/unit'
+
+class MiniTestQueueRunner < MiniTest::Unit
+  def _run_suites(*)
+    self.class.output = $stdout
+    super
+  end
+
+  def _run_anything(*)
+    ret = super
+    output.puts
+    ret
+  end
+
+  def _run_suite(suite, type)
+    output.print '    '
+    output.print suite
+    output.print ': '
+
+    start = Time.now
+    ret = super
+    diff = Time.now - start
+
+    output.puts("  <%.3f>" % diff)
+    ret
+  end
+
+  self.runner = self.new
+  self.output = StringIO.new
+end
+
+class MiniTest::Unit::TestCase
+  class << self
+    attr_accessor :test_suites
+
+    def original_test_suites
+      @@test_suites.keys.reject{ |s| s.test_methods.empty? }
+    end
+  end
+end
+
+module TestQueue
+  class Runner
+    class MiniTest < Runner
+      def initialize
+        super(::MiniTest::Unit::TestCase.original_test_suites.sort_by{ |s| -(stats[s.to_s] || 0) })
+      end
+
+      def run_worker(iterator)
+        ::MiniTest::Unit::TestCase.test_suites = iterator
+        ::MiniTest::Unit.new.run
+      end
+
+      def summarize_worker(worker)
+        worker.stats.each do |s, val|
+          stats[s.to_s] = val
+        end
+
+        num_tests = worker.lines.grep(/ errors?, /).first
+        failures  = worker.lines.select{ |line|
+          line if (line =~ /^Finished/) ... (line =~ / errors?, /)
+        }[1..-2].join("\n").strip
+
+        [ num_tests, failures ]
+      end
+    end
+  end
+end

data/lib/test_queue/runner/rspec.rb

@@ -0,0 +1,65 @@
+require 'test_queue/runner'
+require 'rspec/core'
+
+module RSpec::Core
+  class QueueRunner < CommandLine
+    def initialize
+      super(ARGV)
+    end
+
+    def example_groups
+      @options.configure(@configuration)
+      @configuration.load_spec_files
+      @world.announce_filters
+      @world.example_groups
+    end
+
+    def run_each(iterator)
+      @configuration.error_stream = $stderr
+      @configuration.output_stream = $stdout
+
+      @configuration.reporter.report(0, @configuration.randomize? ? @configuration.seed : nil) do |reporter|
+        begin
+          @configuration.run_hook(:before, :suite)
+          iterator.map {|g|
+            print "    #{g.description}: "
+            start = Time.now
+            ret = g.run(reporter)
+            diff = Time.now-start
+            puts("  <%.3f>" % diff)
+
+            ret
+          }.all? ? 0 : @configuration.failure_exit_code
+        ensure
+          @configuration.run_hook(:after, :suite)
+        end
+      end
+    end
+  end
+end
+
+module TestQueue
+  class Runner
+    class RSpec < Runner
+      def initialize
+        @rspec = ::RSpec::Core::QueueRunner.new
+        super(@rspec.example_groups.sort_by{ |s| -(stats[s.description] || 0) })
+      end
+
+      def run_worker(iterator)
+        @rspec.run_each(iterator)
+      end
+
+      def summarize_worker(worker)
+        worker.stats.each do |s, val|
+          stats[s.description] = val
+        end
+
+        num_tests = worker.lines.grep(/ examples?, /).first
+        failures  = worker.output[/^Failures:\n\n(.*)\n^Finished/m, 1]
+
+        [ num_tests, failures ]
+      end
+    end
+  end
+end

data/lib/test_queue/runner/sample.rb

@@ -0,0 +1,78 @@
+require 'test_queue'
+require 'test_queue/runner'
+
+module TestQueue
+  class Runner
+    class Sample < Runner
+      def spawn_workers
+        puts "Spawning #@concurrency workers"
+        super
+      end
+
+      def after_fork(num)
+        puts "  -- worker #{num} booted as pid #{$$}"
+        super
+      end
+
+      def run_worker(iterator)
+        sum = 0
+        iterator.each do |item|
+          puts "  #{item.inspect}"
+          sum += item
+        end
+        sum
+      end
+
+      def summarize_worker(worker)
+        stats.update(worker.stats)
+
+        summary  = worker.output.scan(/^\s*(\d+)/).join(', ')
+        failures = ''
+
+        [ summary, failures ]
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  TestQueue::Runner::Sample.new(Array(1..10)).execute
+end
+
+__END__
+
+Spawning 4 workers
+  -- worker 0 booted as pid 40406
+  -- worker 1 booted as pid 40407
+  -- worker 2 booted as pid 40408
+  -- worker 3 booted as pid 40409
+
+==> Starting ruby test-queue worker [1] (40407)
+
+  2
+  5
+  8
+
+==> Starting ruby test-queue worker [3] (40409)
+
+
+==> Starting ruby test-queue worker [2] (40408)
+
+  3
+  6
+  9
+
+==> Starting ruby test-queue worker [0] (40406)
+
+  1
+  4
+  7
+  10
+
+==> Summary
+
+    [1]                                                 2, 5, 8      in 0.0024s      (pid 40407 exit 15)
+    [3]                                                              in 0.0036s      (pid 40409 exit 0)
+    [2]                                                 3, 6, 9      in 0.0038s      (pid 40408 exit 18)
+    [0]                                             1, 4, 7, 10      in 0.0044s      (pid 40406 exit 22)
+

data/test-queue.gemspec

@@ -0,0 +1,20 @@
+spec = Gem::Specification.new do |s|
+  s.name = 'test-queue'
+  s.version = '0.1.0'
+  s.summary = 'parallel test runner'
+
+  s.homepage = "http://github.com/tmm1/test-queue"
+
+  s.authors = ["Aman Gupta"]
+  s.email = "ruby@tmm1.net"
+
+  s.has_rdoc = false
+  s.bindir = 'bin'
+  s.executables << 'rspec-queue'
+  s.executables << 'minitest-queue'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'minitest'
+
+  s.files = `git ls-files`.split("\n")
+end

data/test/sample_spec.rb

@@ -0,0 +1,21 @@
+require 'rspec'
+
+describe 'RSpecEqual' do
+  it 'checks equality' do
+    1.should equal(1)
+  end
+end
+
+describe 'RSpecSleep' do
+  it 'sleeps' do
+    start = Time.now
+    sleep 0.25
+    (Time.now-start).should be_within(0.02).of(0.25)
+  end
+end
+
+describe 'RSpecFailure' do
+  it 'fails' do
+    :foo.should eq(:bar)
+  end
+end

data/test/sample_test.rb

@@ -0,0 +1,21 @@
+require 'minitest/unit'
+
+class MiniTestEqual < MiniTest::Unit::TestCase
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+class MiniTestSleep < MiniTest::Unit::TestCase
+  def test_sleep
+    start = Time.now
+    sleep 0.25
+    assert_in_delta Time.now-start, 0.25, 0.02
+  end
+end
+
+class MiniTestFailure < MiniTest::Unit::TestCase
+  def test_fail
+    assert_equal 0, 1
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: test-queue
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Aman Gupta
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-04-24 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: minitest
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: 
+email: ruby@tmm1.net
+executables: 
+- rspec-queue
+- minitest-queue
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- README.md
+- bin/minitest-queue
+- bin/rspec-queue
+- lib/test-queue.rb
+- lib/test_queue.rb
+- lib/test_queue/iterator.rb
+- lib/test_queue/runner.rb
+- lib/test_queue/runner/minitest.rb
+- lib/test_queue/runner/rspec.rb
+- lib/test_queue/runner/sample.rb
+- test-queue.gemspec
+- test/sample_spec.rb
+- test/sample_test.rb
+homepage: http://github.com/tmm1/test-queue
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: parallel test runner
+test_files: []
+
+has_rdoc: false