test-queue-split 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 109d87b665a30d8712019bf2812a5a14fd2daefe
+  data.tar.gz: 2dd5c3c1671ff672832fd92f9b970f808bdf50e7
+SHA512:
+  metadata.gz: d153890bea9cfadef47dc1b7645175ff90a62693a2c3db809af20c311016490571f3bebc1cdafa5d93628709dbff3021f88cfcebb300216d66f64450e7280475
+  data.tar.gz: 30dc8854757eab9b6e82161a1506bd6370a2ef58c2f3e63b3916d6c74d548d6fe7ae4deeefb76db210456d5efdc8e8a431e484b8ec26405d71f8b0afb3060686

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'minitest', '5.3.3'

data/Gemfile-minitest4

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'minitest', '4.7.3'

data/Gemfile-minitest4.lock

@@ -0,0 +1,38 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.2.13)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cucumber (1.3.15)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.1)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    minitest (4.7.3)
+    multi_json (1.10.1)
+    multi_test (0.1.1)
+    rspec (2.99.0)
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
+    rspec-core (2.99.1)
+    rspec-expectations (2.99.1)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber (~> 1.3.10)
+  minitest (= 4.7.3)
+  rspec (>= 2.13, < 4.0)
+  test-queue!

data/Gemfile-rspec3-0

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'rspec', '~> 3.0.0'

data/Gemfile-rspec3-0.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.2.12)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cucumber (1.3.19)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    multi_json (1.11.0)
+    multi_test (0.1.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.4)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber (~> 1.3.10)
+  rspec (~> 3.0.0)
+  test-queue!

data/Gemfile-rspec3-1

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'rspec', '~> 3.1.0'

data/Gemfile-rspec3-1.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.2.12)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cucumber (1.3.19)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    multi_json (1.11.0)
+    multi_test (0.1.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber (~> 1.3.10)
+  rspec (~> 3.1.0)
+  test-queue!

data/Gemfile-rspec3-2

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'rspec', '~> 3.2.0'

data/Gemfile-rspec3-2.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.2.12)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cucumber (1.3.19)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    multi_json (1.11.0)
+    multi_test (0.1.2)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber (~> 1.3.10)
+  rspec (~> 3.2.0)
+  test-queue!

data/Gemfile-testunit

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem 'test-unit', '3.0.8'

data/Gemfile-testunit.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    test-queue (0.2.13)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cucumber (1.3.18)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.1)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    multi_json (1.10.1)
+    multi_test (0.1.1)
+    power_assert (0.2.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    test-unit (3.0.8)
+      power_assert
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cucumber (~> 1.3.10)
+  rspec (>= 2.13, < 4.0)
+  test-queue!
+  test-unit (= 3.0.8)

data/Gemfile.lock

@@ -0,0 +1,54 @@
+PATH
+  remote: .
+  specs:
+    test-queue-split (0.3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    appium_thor (1.0.1)
+      posix-spawn (~> 0.3, >= 0.3.8)
+      redcarpet (~> 3.1, >= 3.1.2)
+      yard (~> 0.8, >= 0.8.7.4)
+    builder (3.2.2)
+    cucumber (1.3.20)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.1.2)
+    diff-lcs (1.2.5)
+    gherkin (2.12.2)
+      multi_json (~> 1.3)
+    minitest (5.3.3)
+    multi_json (1.11.2)
+    multi_test (0.1.2)
+    posix-spawn (0.3.11)
+    redcarpet (3.3.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  appium_thor (~> 1.0.1)
+  cucumber (~> 1.3.10)
+  minitest (= 5.3.3)
+  rspec (>= 2.13, < 4.0)
+  test-queue-split!
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -0,0 +1,114 @@
+## test-queue
+
+Yet another parallel test runner, built using a centralized queue to ensure
+optimal distribution of tests between workers.
+
+Specifically optimized for CI environments: build statistics from each run
+are stored locally and used to sort the queue at the beginning of the
+next run.
+
+### design
+
+test-queue uses a simple master + pre-fork worker model. The master
+exposes a unix domain socket server which workers use to grab tests off
+the queue.
+
+```
+─┬─ 21232 minitest-queue master
+ ├─── 21571 minitest-queue worker [3] - AuthenticationTest
+ ├─── 21568 minitest-queue worker [2] - ApiTest
+ ├─── 21565 minitest-queue worker [1] - UsersControllerTest
+ └─── 21562 minitest-queue worker [0] - UserTest
+```
+
+test-queue also has a distributed mode, where additional masters can share
+the workload and relay results back to a central master.
+
+### environment variables
+
+- `TEST_QUEUE_WORKERS`: number of workers to use per master (default: all available cores)
+- `TEST_QUEUE_VERBOSE`: show results as they are available (default: `0`)
+- `TEST_QUEUE_SOCKET`: unix socket `path` (or tcp `address:port` pair) used for communication (default: `/tmp/test_queue_XXXXX.sock`)
+- `TEST_QUEUE_RELAY`: relay results back to a central master, specified as tcp `address:port`
+- `TEST_QUEUE_STATS`: `path` to cache build stats in-build CI runs (default: `.test_queue_stats`)
+- `TEST_QUEUE_FORCE`: comma separated list of suites to run
+- `TEST_QUEUE_RELAY_TIMEOUT`: when using remote workers, the amount of time a worker will try to reconnect to start work
+- `TEST_QUEUE_RELAY_TOKEN`: when using remote workers, this must be the same on both workers and the server for remote workers to run tests.
+- `TEST_QUEUE_SLAVE_MESSAGE`: when using remote workers, set this on a slave worker and it will appear on the slave's connection message on the master.
+
+### usage
+
+test-queue bundles `testunit-queue`, `minitest-queue` and `rspec-queue` binaries which can be used directly:
+
+```
+$ minitest-queue $(find test/ -name \*_test.rb)
+$ rspec-queue --format progress spec
+```
+
+But the underlying `TestQueue::Runner::TestUnit`, `TestQueue::Runner::MiniTest` and `TestQueue::Runner::RSpec` are
+built to be subclassed by your application. I recommend checking a new
+executable into your project using one of these superclasses.
+
+```
+$ vim script/test-queue
+$ chmod +x script/test-queue
+$ git add script/test-queue
+```
+
+Since test-queue uses `fork(2)` to spawn off workers, you must ensure each worker
+runs in an isolated environment. Use the `after_fork` hook with a custom
+runner to reset any global state.
+
+``` ruby
+#!/usr/bin/env ruby
+
+class MyAppTestRunner < TestQueue::Runner::MiniTest
+  def after_fork(num)
+    # use separate mysql database (we assume it exists and has the right schema already)
+    ActiveRecord::Base.configurations['test']['database'] << num.to_s
+    ActiveRecord::Base.establish_connection(:test)
+
+    # use separate redis database
+    $redis.client.db = num
+    $redis.client.reconnect
+  end
+
+  def prepare(concurrency)
+    # create mysql databases exists with correct schema
+    concurrency.times do |i|
+      # ...
+    end
+
+    # If this is a remote slave, tell the master something about us
+    @slave_message = "Output for slave 123: http://myhost.com/build/123"
+  end
+
+  def around_filter(suite)
+    $stats.timing("test.#{suite}.runtime") do
+      yield
+    end
+  end
+end
+
+MyAppTestRunner.new.execute
+```
+
+### distributed mode
+
+To use distributed mode, the central master must listen on a tcp port. Additional masters can be booted
+in relay mode to connect to the central master. Workers must provide a TEST_QUEUE_RELAY_TOKEN to match
+the master's.
+
+```
+$ TEST_QUEUE_RELAY_TOKEN=123 TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec minitest-queue ./test/sample_test.rb
+$ TEST_QUEUE_RELAY_TOKEN=123 TEST_QUEUE_RELAY=0.0.0.0:12345  bundle exec minitest-queue ./test/sample_test.rb
+$ TEST_QUEUE_RELAY_TOKEN=123 ./test-multi.sh
+```
+
+See the [Parameterized Trigger Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Parameterized+Trigger+Plugin)
+for a simple way to do this with jenkins.
+
+### see also
+
+  * https://github.com/Shopify/rails_parallel
+  * https://github.com/grosser/parallel_tests

data/Thorfile

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'appium_thor'
+
+Appium::Thor::Config.set do
+  gem_name 'test-queue-split'
+  github_owner 'sbonebrake'
+  github_name 'test-queue'
+  version_file 'lib/test_queue/version.rb'
+end

data/bin/cucumber-queue

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/cucumber'
+TestQueue::Runner::Cucumber.new.execute

data/bin/minitest-queue

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/minitest'
+ARGV.each{ |f| require(f) }
+TestQueue::Runner::MiniTest.new.execute

data/bin/rspec-queue

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/rspec'
+TestQueue::Runner::RSpec.new.execute

data/bin/testunit-queue

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'test_queue'
+require 'test_queue/runner/testunit'
+ARGV.each{ |f| require(f) }
+TestQueue::Runner::TestUnit.new.execute

data/features/bad.feature

@@ -0,0 +1,5 @@
+Feature: bad
+  Scenario: failure
+    Given a
+    When bad
+    Then c

data/features/sample.feature

@@ -0,0 +1,25 @@
+Feature: Foobar
+  Scenario: Bar
+    Given a
+    When b
+    Then c
+  Scenario: Baz
+    Given a
+    When b
+    Then c
+  Scenario: Bam
+    Given a
+    When b
+    Then c
+  Scenario: Car
+    Given a
+    When b
+    Then c
+  Scenario: Caz
+    Given a
+    When b
+    Then c
+  Scenario: Cam
+    Given a
+    When b
+    Then c

data/features/sample2.feature

@@ -0,0 +1,29 @@
+Feature: Foobar2
+  Scenario: Bar
+    Given a
+    When b
+    Then c
+  Scenario: Baz
+    Given a
+    When b
+    Then c
+  Scenario: Bam
+    Given a
+    When b
+    Then c
+  Scenario: Car
+    Given a
+    When b
+    Then c
+  Scenario: Caz
+    Given a
+    When b
+    Then c
+  Scenario: Cam
+    Given a
+    When b
+    Then c
+  Scenario: failure
+    Given a
+    When bad
+    Then c

data/features/step_definitions/common.rb

@@ -0,0 +1,15 @@
+Given(/^a$/) do
+  sleep 0.10
+end
+
+When(/^b$/) do
+  sleep 0.25
+end
+
+When(/^bad$/) do
+  1.should == 0
+end
+
+Then(/^c$/) do
+  1.should == 1
+end

data/lib/test-queue.rb

@@ -0,0 +1 @@
+require 'test_queue'

data/lib/test_queue/iterator.rb

@@ -0,0 +1,72 @@
+module TestQueue
+  class Iterator
+    attr_reader :stats, :sock
+
+    def initialize(sock, suites, filter=nil)
+      @done = false
+      @stats = {}
+      @procline = $0
+      @sock = sock
+      @suites = suites
+      @filter = filter
+      if @sock =~ /^(.+):(\d+)$/
+        @tcp_address = $1
+        @tcp_port = $2.to_i
+      end
+    end
+
+    def each
+      fail "already used this iterator. previous caller: #@done" if @done
+
+      while true
+        client = connect_to_master('POP')
+        break if client.nil?
+        r, w, e = IO.select([client], nil, [client], nil)
+        break if !e.empty?
+
+        if data = client.read(65536)
+          client.close
+          item = Marshal.load(data)
+          break if item.nil? || item.empty?
+          suite = @suites[item]
+
+          $0 = "#{@procline} - #{suite.respond_to?(:description) ? suite.description : suite}"
+          start = Time.now
+          if @filter
+            @filter.call(suite){ yield suite }
+          else
+            yield suite
+          end
+          @stats[suite.to_s] = Time.now - start
+        else
+          break
+        end
+      end
+    rescue Errno::ENOENT, Errno::ECONNRESET, Errno::ECONNREFUSED
+    ensure
+      @done = caller.first
+      File.open("/tmp/test_queue_worker_#{$$}_stats", "wb") do |f|
+        f.write Marshal.dump(@stats)
+      end
+    end
+
+    def connect_to_master(cmd)
+      sock =
+        if @tcp_address
+          TCPSocket.new(@tcp_address, @tcp_port)
+        else
+          UNIXSocket.new(@sock)
+        end
+      sock.puts(cmd)
+      sock
+    rescue Errno::EPIPE
+      nil
+    end
+
+    include Enumerable
+
+    def empty?
+      false
+    end
+  end
+end

data/lib/test_queue/runner/cucumber.rb

@@ -0,0 +1,46 @@
+require 'cucumber'
+require 'cucumber/rspec/disable_option_parser'
+require 'cucumber/cli/main'
+
+module Cucumber
+  module Ast
+    class Features
+      attr_accessor :features
+    end
+
+    class Feature
+      def to_s
+        title
+      end
+    end
+  end
+end
+
+module TestQueue
+  class Runner
+    class Cucumber < Runner
+      def initialize
+        @cli = ::Cucumber::Cli::Main.new(ARGV.dup)
+        @runtime = ::Cucumber::Runtime.new(@cli.configuration)
+        @features_loader = @runtime.send(:features)
+        features = @features_loader.features.sort_by{ |s| -(stats[s.to_s] || 0) }
+        super(features)
+      end
+
+      def run_worker(iterator)
+        @features_loader.features = iterator
+        @cli.execute!(@runtime)
+      end
+
+      def summarize_worker(worker)
+        worker.stats.each do |s, val|
+          stats[s.to_s] = val
+        end
+
+        output = worker.output.gsub(/\e\[\d+./,'')
+        worker.summary  = output.split("\n").grep(/^\d+ (scenarios?|steps?)/).first
+        worker.failure_output = output.scan(/^Failing Scenarios:\n(.*)\n\d+ scenarios?/m).join("\n")
+      end
+    end
+  end
+end

data/lib/test_queue/runner/minitest.rb

@@ -0,0 +1,25 @@
+begin
+  require 'minitest'
+  require 'test_queue/runner/minitest5'
+rescue LoadError => e
+  require 'minitest/unit'
+  require 'test_queue/runner/minitest4'
+end
+
+module TestQueue
+  class Runner
+    class MiniTest < Runner
+      def summarize_worker(worker)
+        worker.stats.each do |s, val|
+          stats[s.to_s] = val
+        end
+
+        worker.summary = worker.lines.grep(/, \d+ errors?, /).first
+        failures  = worker.lines.select{ |line|
+          line if (line =~ /^Finished/) ... (line =~ /, \d+ errors?, /)
+        }[1..-2]
+        worker.failure_output = failures.join("\n") if failures
+      end
+    end
+  end
+end