test-queue 0.1.3 → 0.2.0.beta.1

data/README.md

@@ -7,13 +7,6 @@ Specifically optimized for CI environments: build statistics from each run
 are stored locally and used to sort the queue at the beginning of the
 next run.
 
-### usage
-
-```
-$ minitest-queue $(find test/ -name \*_test.rb)
-$ rspec-queue --format progress spec
-```
-
 ### design
 
 test-queue uses a simple master + pre-fork worker model. The master
@@ -28,17 +21,45 @@ the queue.
  └─── 21562 minitest-queue worker [0] - UserTest
 ```
 
-### customization
+test-queue also has a distributed mode, where additional masters can share
+the workload and relay results back to a central master.
+
+### environment variables
+
+- `TEST_QUEUE_WORKERS`: number of workers to use per master (default: all available cores)
+- `TEST_QUEUE_VERBOSE`: show results as they are available (default: `0`)
+- `TEST_QUEUE_SOCKET`: unix socket `path` (or tcp `address:port` pair) used for communication (default: `/tmp/test_queue_XXXXX.sock`)
+- `TEST_QUEUE_RELAY`: relay results back to a central master, specified as tcp `address:port`
+- `TEST_QUEUE_STATS`: `path` to cache build stats in-build CI runs (default: `.test_queue_stats`)
+
+### usage
+
+test-queue bundles `minitest-queue` and `rspec-queue` binaries which can be used directly:
+
+```
+$ minitest-queue $(find test/ -name \*_test.rb)
+$ rspec-queue --format progress spec
+```
+
+But the underlying `TestQueue::Runner::MiniTest` and `TestQueue::Runner::Rspec` are
+built to be subclassed by your application. I recommend checking a new
+executable into your project using one of these superclasses.
+
+```
+$ vim script/test-queue
+$ chmod +x script/test-queue
+$ git add script/test-queue
+```
 
 Since test-queue uses `fork(2)` to spawn off workers, you must ensure each worker
 runs in an isolated environment. Use the `after_fork` hook with a custom
-runner to reset any global state:
+runner to reset any global state.
 
 ``` ruby
-class CustomMiniTestRunner < TestQueue::Runner::MiniTest
-  def after_fork(num)
-    super
+#!/usr/bin/env ruby
 
+class MyAppTestRunner < TestQueue::Runner::MiniTest
+  def after_fork(num)
     # use separate mysql database (we assume it exists and has the right schema already)
     ActiveRecord::Base.configurations['test']['database'] << num.to_s
     ActiveRecord::Base.establish_connection(:test)
@@ -47,11 +68,37 @@ class CustomMiniTestRunner < TestQueue::Runner::MiniTest
     $redis.client.db = num
     $redis.client.reconnect
   end
+
+  def prepare(concurrency)
+    # create mysql databases exists with correct schema
+    concurrency.times do |i|
+      # ...
+    end
+  end
+
+  def around_filter(suite)
+    $stats.timing("test.#{suite}.runtime") do
+      yield
+    end
+  end
 end
 
 CustomMiniTestRunner.new.execute
 ```
 
+### distributed mode
+
+To use distributed mode, the central master must listen on a tcp port. Additional masters can be booted
+in relay mode to connect to the central master.
+
+```
+$ TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec minitest-queue ./test/sample_test.rb
+$ TEST_QUEUE_RELAY=0.0.0.0:12345  bundle exec minitest-queue ./test/sample_test.rb
+```
+
+See the [Parameterized Trigger Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Parameterized+Trigger+Plugin)
+for a simple way to do this with jenkins.
+
 ### see also
 
   * https://github.com/Shopify/rails_parallel

data/lib/test_queue/iterator.rb

@@ -1,30 +1,42 @@
 module TestQueue
   class Iterator
-    attr_reader :stats
+    attr_reader :stats, :sock
 
-    def initialize(sock)
-      @sock = sock
+    def initialize(sock, suites, filter=nil)
       @done = false
       @stats = {}
       @procline = $0
+      @sock = sock
+      @suites = suites
+      @filter = filter
+      if @sock =~ /^(.+):(\d+)$/
+        @tcp_address = $1
+        @tcp_port = $2.to_i
+      end
     end
 
     def each
       fail 'already used this iterator' if @done
 
       while true
-        client = UNIXSocket.new(@sock)
+        client = connect_to_master('POP')
         r, w, e = IO.select([client], nil, [client], nil)
         break if !e.empty?
 
-        if data = client.read(16384)
+        if data = client.read(65536)
           client.close
           item = Marshal.load(data)
-          $0 = "#{@procline} - #{item.respond_to?(:description) ? item.description : item}"
+          break if item.nil?
+          suite = @suites[item]
 
+          $0 = "#{@procline} - #{suite.respond_to?(:description) ? suite.description : suite}"
           start = Time.now
-          yield item
-          @stats[item] = Time.now - start
+          if @filter
+            @filter.call(suite){ yield suite }
+          else
+            yield suite
+          end
+          @stats[suite.to_s] = Time.now - start
         else
           break
         end
@@ -37,6 +49,17 @@ module TestQueue
       end
     end
 
+    def connect_to_master(cmd)
+      sock =
+        if @tcp_address
+          TCPSocket.new(@tcp_address, @tcp_port)
+        else
+          UNIXSocket.new(@sock)
+        end
+      sock.puts(cmd)
+      sock
+    end
+
     include Enumerable
 
     def empty?

data/lib/test_queue/runner.rb

@@ -3,7 +3,7 @@ require 'fileutils'
 
 module TestQueue
   class Worker
-    attr_accessor :pid, :status, :output, :stats, :num
+    attr_accessor :pid, :status, :output, :stats, :num, :host
     attr_accessor :start_time, :end_time
 
     def initialize(pid, num)
@@ -11,6 +11,7 @@ module TestQueue
       @num = num
       @start_time = Time.now
       @output = ''
+      @stats = {}
     end
 
     def lines
@@ -21,10 +22,12 @@ module TestQueue
   class Runner
     attr_accessor :concurrency
 
-    def initialize(queue, concurrency=nil)
+    def initialize(queue, concurrency=nil, socket=nil, relay=nil)
       raise ArgumentError, 'array required' unless Array === queue
 
+      @procline = $0
       @queue = queue
+      @suites = queue.inject(Hash.new){ |hash, suite| hash.update suite.to_s => suite }
 
       @workers = {}
       @completed = []
@@ -39,11 +42,27 @@ module TestQueue
         else
           2
         end
+
+      @socket =
+        socket ||
+        ENV['TEST_QUEUE_SOCKET'] ||
+        "/tmp/test_queue_#{$$}_#{object_id}.sock"
+
+      @relay =
+        relay ||
+        ENV['TEST_QUEUE_RELAY']
+
+      if @relay == @socket
+        STDERR.puts "*** Detected TEST_QUEUE_RELAY == TEST_QUEUE_SOCKET. Disabling relay mode."
+        @relay = nil
+      elsif @relay
+        @queue = []
+      end
     end
 
     def stats
       @stats ||=
-        if File.exists?(file = '.test_queue_stats')
+        if File.exists?(file = stats_file)
           Marshal.load(IO.binread(file)) || {}
         else
           {}
@@ -52,13 +71,18 @@ module TestQueue
 
     def execute
       $stdout.sync = $stderr.sync = true
+      @start_time = Time.now
 
       @concurrency > 0 ?
         execute_parallel :
         execute_sequential
     ensure
+      summarize_internal
+    end
+
+    def summarize_internal
       puts
-      puts "==> Summary"
+      puts "==> Summary (#{@completed.size} workers in %.4fs)" % (Time.now-@start_time)
       puts
 
       @failures = ''
@@ -66,12 +90,14 @@ module TestQueue
         summary, failures = summarize_worker(worker)
         @failures << failures if failures
 
-        puts "    [%2d] %55s      in %.4fs      (pid %d exit %d)" % [
+        puts "    [%2d] %60s      %4d suites in %.4fs      (pid %d exit %d%s)" % [
           worker.num,
           summary,
+          worker.stats.size,
           worker.end_time - worker.start_time,
           worker.pid,
-          worker.status.exitstatus
+          worker.status.exitstatus,
+          worker.host && " on #{worker.host.split('.').first}"
         ]
       end
 
@@ -85,14 +111,23 @@ module TestQueue
       puts
 
       if @stats
-        File.open('.test_queue_stats', 'wb') do |f|
+        File.open(stats_file, 'wb') do |f|
           f.write Marshal.dump(stats)
         end
       end
 
+      summarize
       exit! @completed.inject(0){ |s, worker| s + worker.status.exitstatus }
     end
 
+    def summarize
+    end
+
+    def stats_file
+      ENV['TEST_QUEUE_STATS'] ||
+      '.test_queue_stats'
+    end
+
     def execute_sequential
       exit! run_worker(@queue)
     end
@@ -114,32 +149,59 @@ module TestQueue
     end
 
     def start_master
-      @socket = "/tmp/test_queue_#{$$}_#{object_id}.sock"
-      FileUtils.rm(@socket) if File.exists?(@socket)
-      @server = UNIXServer.new(@socket)
+      if relay?
+        begin
+          sock = connect_to_relay
+          sock.puts("SLAVE #{@concurrency}")
+          sock.close
+        rescue Errno::ECONNREFUSED
+          STDERR.puts "*** Unable to connect to relay #{@relay}. Aborting.."
+          exit! 1
+        end
+      else
+        if @socket =~ /^(?:(.+):)?(\d+)$/
+          address = $1 || '0.0.0.0'
+          port = $2.to_i
+          @socket = "#$1:#$2"
+          @server = TCPServer.new(address, port)
+        else
+          FileUtils.rm(@socket) if File.exists?(@socket)
+          @server = UNIXServer.new(@socket)
+        end
+      end
+
+      desc = "test-queue master (#{relay?? "relaying to #{@relay}" : @socket})"
+      puts "Starting #{desc}"
+      $0 = "#{desc} - #{@procline}"
     end
 
     def stop_master
-      FileUtils.rm_f(@socket) if @socket
+      return if relay?
+
+      FileUtils.rm_f(@socket) if @socket && @server.is_a?(UNIXServer)
       @server.close rescue nil if @server
       @socket = @server = nil
     end
 
     def spawn_workers
+      prepare(@concurrency)
+
       @concurrency.times do |i|
         num = i+1
 
         pid = fork do
-          @server.close
-          after_fork(num)
-          exit! run_worker(iterator = Iterator.new(@socket)) || 0
+          @server.close if @server
+
+          iterator = Iterator.new(relay?? @relay : @socket, @suites, method(:around_filter))
+          after_fork_internal(num, iterator)
+          exit! run_worker(iterator) || 0
         end
 
         @workers[pid] = Worker.new(pid, num)
       end
     end
 
-    def after_fork(num)
+    def after_fork_internal(num, iterator)
       srand
 
       output = File.open("/tmp/test_queue_worker_#{$$}_output", 'w')
@@ -150,8 +212,20 @@ module TestQueue
 
       $0 = "test-queue worker [#{num}]"
       puts
-      puts "==> Starting #$0 (#{Process.pid})"
+      puts "==> Starting #$0 (#{Process.pid}) - iterating over #{iterator.sock}"
       puts
+
+      after_fork(num)
+    end
+
+    def prepare(concurrency)
+    end
+
+    def around_filter(suite)
+      yield
+    end
+
+    def after_fork(num)
     end
 
     def run_worker(iterator)
@@ -169,15 +243,13 @@ module TestQueue
       [ num_tests, failures ]
     end
 
-    def cleanup_worker
-      if pid = Process.waitpid and worker = @workers.delete(pid)
-        @completed << worker
+    def cleanup_worker(blocking=true)
+      if pid = Process.waitpid(-1, blocking ? 0 : Process::WNOHANG) and worker = @workers.delete(pid)
         worker.status = $?
         worker.end_time = Time.now
 
         if File.exists?(file = "/tmp/test_queue_worker_#{pid}_output")
           worker.output = IO.binread(file)
-          puts worker.output
           FileUtils.rm(file)
         end
 
@@ -185,16 +257,43 @@ module TestQueue
           worker.stats = Marshal.load(IO.binread(file))
           FileUtils.rm(file)
         end
+
+        relay_to_master(worker) if relay?
+        worker_completed(worker)
       end
     end
 
+    def worker_completed(worker)
+      @completed << worker
+      puts worker.output if ENV['TEST_QUEUE_VERBOSE']
+    end
+
     def distribute_queue
-      until @queue.empty?
-        IO.select([@server], nil, nil, nil)
+      return if relay?
+      remote_workers = 0
 
-        sock = @server.accept
-        sock.write(Marshal.dump(@queue.shift))
-        sock.close
+      until @queue.empty? && remote_workers == 0
+        if IO.select([@server], nil, nil, 0.1).nil?
+          cleanup_worker(false) # check for worker deaths
+        else
+          sock = @server.accept
+          cmd = sock.gets.strip
+          case cmd
+          when 'POP'
+            data = Marshal.dump(@queue.shift.to_s)
+            sock.write(data)
+          when /^SLAVE (\d+)/
+            num = $1.to_i
+            remote_workers += num
+            STDERR.puts "*** slave connected with additional #{num} workers"
+          when /^WORKER (\d+)/
+            data = sock.read($1.to_i)
+            worker = Marshal.load(data)
+            worker_completed(worker)
+            remote_workers -= 1
+          end
+          sock.close
+        end
       end
     ensure
       stop_master
@@ -203,5 +302,24 @@ module TestQueue
         cleanup_worker
       end
     end
+
+    def relay?
+      !!@relay
+    end
+
+    def connect_to_relay
+      TCPSocket.new(*@relay.split(':'))
+    end
+
+    def relay_to_master(worker)
+      worker.host = Socket.gethostname
+      data = Marshal.dump(worker)
+
+      sock = connect_to_relay
+      sock.puts("WORKER #{data.bytesize}")
+      sock.write(data)
+    ensure
+      sock.close if sock
+    end
   end
 end

data/lib/test_queue/runner/minitest.rb

@@ -45,7 +45,8 @@ module TestQueue
   class Runner
     class MiniTest < Runner
       def initialize
-        super(::MiniTest::Unit::TestCase.original_test_suites.sort_by{ |s| -(stats[s.to_s] || 0) })
+        tests = ::MiniTest::Unit::TestCase.original_test_suites.sort_by{ |s| -(stats[s.to_s] || 0) }
+        super(tests)
       end
 
       def run_worker(iterator)
@@ -61,7 +62,8 @@ module TestQueue
         num_tests = worker.lines.grep(/ errors?, /).first
         failures  = worker.lines.select{ |line|
           line if (line =~ /^Finished/) ... (line =~ / errors?, /)
-        }[1..-2].join("\n")
+        }[1..-2]
+        failures = failures.join("\n") if failures
 
         [ num_tests, failures ]
       end

data/test-queue.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'test-queue'
-  s.version = '0.1.3'
+  s.version = '0.2.0.beta.1'
   s.summary = 'parallel test runner'
 
   s.homepage = "http://github.com/tmm1/test-queue"

data/test/sample_minispec.rb

@@ -0,0 +1,31 @@
+require 'minitest/spec'
+
+class Meme
+  def i_can_has_cheezburger?
+    "OHAI!"
+  end
+
+  def will_it_blend?
+    "YES!"
+  end
+end
+
+describe Meme do
+  before do
+    @meme = Meme.new
+  end
+
+  describe "when asked about cheeseburgers" do
+    it "must respond positively" do
+      sleep 0.1
+      @meme.i_can_has_cheezburger?.must_equal "OHAI!"
+    end
+  end
+
+  describe "when asked about blending possibilities" do
+    it "won't say no" do
+      sleep 0.1
+      @meme.will_it_blend?.wont_match /^no/i
+    end
+  end
+end

data/test/sample_spec.rb

@@ -6,11 +6,13 @@ describe 'RSpecEqual' do
   end
 end
 
-describe 'RSpecSleep' do
-  it 'sleeps' do
-    start = Time.now
-    sleep 0.25
-    (Time.now-start).should be_within(0.02).of(0.25)
+50.times do |i|
+  describe "RSpecSleep(#{i})" do
+    it "sleeps" do
+      start = Time.now
+      sleep(0.25)
+      (Time.now-start).should be_within(0.02).of(0.25)
+    end
   end
 end
 

data/test/sample_test.rb

@@ -6,12 +6,14 @@ class MiniTestEqual < MiniTest::Unit::TestCase
   end
 end
 
-class MiniTestSleep < MiniTest::Unit::TestCase
-  def test_sleep
-    start = Time.now
-    sleep 0.25
-    assert_in_delta Time.now-start, 0.25, 0.02
-  end
+5.times do |i|
+  Object.const_set("MiniTestSleep#{i}", Class.new(MiniTest::Unit::TestCase) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
 end
 
 class MiniTestFailure < MiniTest::Unit::TestCase

metadata

@@ -1,63 +1,56 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: test-queue
-version: !ruby/object:Gem::Version 
-  hash: 29
-  prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 3
-  version: 0.1.3
+version: !ruby/object:Gem::Version
+  version: 0.2.0.beta.1
+  prerelease: 6
 platform: ruby
-authors: 
+authors:
 - Aman Gupta
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-04-29 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 25
-        segments: 
-        - 2
-        - 13
-        version: "2.13"
+      - !ruby/object:Gem::Version
+        version: '2.13'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: minitest
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 37
-        segments: 
-        - 4
-        - 7
-        - 3
+      - !ruby/object:Gem::Version
         version: 4.7.3
   type: :development
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.7.3
 description: 
 email: ruby@tmm1.net
-executables: 
+executables:
 - rspec-queue
 - minitest-queue
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - Gemfile
 - Gemfile.lock
 - README.md
@@ -72,41 +65,32 @@ files:
 - lib/test_queue/runner/rspec.rb
 - lib/test_queue/runner/sample.rb
 - test-queue.gemspec
+- test/sample_minispec.rb
 - test/sample_spec.rb
 - test/sample_test.rb
 homepage: http://github.com/tmm1/test-queue
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: parallel test runner
 test_files: []
-
 has_rdoc: false