test-queue 0.2.13 → 0.3.0

data/script/bootstrap

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -ex
+
+cd "$(dirname "$0")/.."
+ROOT=$(pwd)
+
+rm -rf tmp
+mkdir tmp
+mkdir -p vendor/bats
+curl --silent --location --show-error https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz | tar -xz -C tmp
+tmp/bats-0.4.0/install.sh "$ROOT/vendor/bats"

data/script/cibuild

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -ex
+
+if [ "$SUITE" = "ruby" ]; then
+  exec script/spec
+fi
+
+export TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1
+
+if [ -f "Gemfile-$SUITE" ]; then
+  export BUNDLE_GEMFILE="Gemfile-$SUITE"
+else
+  export BUNDLE_GEMFILE=Gemfile
+fi
+
+bundle install
+
+vendor/bats/bin/bats test

data/script/spec

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+export BUNDLE_GEMFILE=Gemfile-rspec3-2
+bundle install
+bundle exec rspec spec

data/spec/stats_spec.rb

@@ -0,0 +1,76 @@
+require "fileutils"
+require "tempfile"
+require "test_queue/stats"
+
+RSpec.describe TestQueue::Stats do
+  before do
+    Tempfile.open("test_queue_stats") do |f|
+      @path = f.path
+      f.close!
+    end
+  end
+
+  after do
+    FileUtils.rm_f(@path)
+  end
+
+  describe "#initialize" do
+    it "ignores empty stats files" do
+      File.write(@path, "")
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores invalid data in the stats files" do
+      File.write(@path, "this is not marshal data")
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores badly-typed data in the stats file" do
+      File.write(@path, Marshal.dump(["heyyy"]))
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores stats files with a wrong version number" do
+      File.write(@path, Marshal.dump({ :version => 1e8, :suites => "boom" }))
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+  end
+
+  it "can save and load data" do
+    stats = TestQueue::Stats.new(@path)
+    time = truncated_now
+    suites = [
+      TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
+      TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time + 5),
+    ]
+    stats.record_suites(suites)
+    stats.save
+
+    stats = TestQueue::Stats.new(@path)
+    expect(stats.all_suites.sort_by(&:name)).to eq(suites)
+  end
+
+  it "prunes suites not seen in the last 8 days" do
+    stats = TestQueue::Stats.new(@path)
+    time = truncated_now
+    suites = [
+      TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
+      TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time - (8 * 24 * 60 * 60) - 2),
+      TestQueue::Stats::Suite.new("Suite3", "baz.rb", 0.6, time - (7 * 24 * 60 * 60)),
+    ]
+    stats.record_suites(suites)
+    stats.save
+
+    stats = TestQueue::Stats.new(@path)
+    expect(stats.all_suites.map(&:name).sort).to eq(%w[Suite1 Suite3])
+  end
+
+  # Returns Time.now rounded down to the nearest second.
+  def truncated_now
+    Time.at(Time.now.to_i)
+  end
+end

data/test-queue.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'test-queue'
-  s.version = '0.2.13'
+  s.version = '0.3.0'
   s.summary = 'parallel test runner'
   s.description = 'minitest/rspec parallel test runner for CI environments'
 
@@ -17,8 +17,5 @@ spec = Gem::Specification.new do |s|
   s.executables << 'testunit-queue'
   s.executables << 'cucumber-queue'
 
-  s.add_development_dependency 'rspec', '>= 2.13', '< 4.0'
-  s.add_development_dependency 'cucumber', '~> 1.3.10'
-
   s.files = `git ls-files`.split("\n")
 end

data/test/cucumber.bats

@@ -0,0 +1,57 @@
+load "testlib"
+
+SCRATCH=tmp/cucumber-tests
+
+setup() {
+  require_gem "cucumber" ">= 1.0"
+  rm -rf $SCRATCH
+  mkdir -p $SCRATCH
+}
+
+teardown() {
+  rm -rf $SCRATCH
+}
+
+@test "cucumber-queue succeeds when all features pass" {
+  run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "cucumber-queue fails when a feature fails" {
+  export FAIL=1
+  run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
+  assert_status 2
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "cucumber test/samples/features/bad.feature:2 # Scenario: failure"
+  assert_output_contains "cucumber test/samples/features/sample2.feature:26 # Scenario: failure"
+}
+
+@test "cucumber-queue fails when given a missing feature" {
+  run bundle exec cucumber-queue test/samples/does_not_exist.feature --require test/samples/features/step_definitions
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed."
+}
+
+@test "cucumber-queue fails when given a malformed feature" {
+  [ -f README.md ]
+  run bundle exec cucumber-queue README.md --require test/samples/features/step_definitions
+
+  # Cucumber 1 and 2 fail in different ways.
+  refute_status 0
+  assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:'
+}
+
+@test "cucumber-queue handles test file being deleted" {
+  cp test/samples/features/*.feature $SCRATCH
+
+  run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
+  assert_status 0
+  assert_output_matches "Feature: Foobar$"
+
+  rm $SCRATCH/sample.feature
+
+  run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
+  assert_status 0
+  refute_output_matches "Feature: Foobar$"
+}

data/test/minitest4.bats

@@ -0,0 +1,34 @@
+load "testlib"
+
+setup() {
+  require_gem "minitest" ">= 4.0"
+}
+
+@test "minitest-queue succeeds when all tests pass" {
+  run bundle exec minitest-queue ./test/samples/*_minitest4.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue fails when a test fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minitest4.rb
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "minitest-queue succeeds when all specs pass" {
+  run bundle exec minitest-queue ./test/samples/*_minispec.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue fails when a spec fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minispec.rb
+  assert_status 1
+  assert_output_contains "1) Failure:"
+  assert_output_contains "Meme::when asked about blending possibilities#test_0002_fails"
+}

data/test/minitest5.bats

@@ -0,0 +1,111 @@
+load "testlib"
+
+SCRATCH=tmp/minitest5-tests
+
+setup() {
+  require_gem "minitest" ">= 5.0"
+  rm -rf $SCRATCH
+  mkdir -p $SCRATCH
+}
+
+teardown() {
+  rm -rf $SCRATCH
+}
+
+@test "minitest-queue (minitest5) succeeds when all tests pass" {
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue (minitest5) fails when a test fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "TEST_QUEUE_FORCE whitelists certain tests" {
+  export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,MiniTestSleep8"
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "MiniTestSleep21"
+  assert_output_contains "MiniTestSleep8"
+  refute_output_contains "MiniTestSleep9"
+}
+
+@test "multi-master succeeds when all tests pass" {
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "multi-master fails when a test fails" {
+  export FAIL=1
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" {
+  export TEST_QUEUE_WORKERS=0
+  run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  assert_status 1
+  assert_output_contains "Worker count (0) must be greater than 0"
+}
+
+@test "minitest-queue fails when given a missing test file" {
+  run bundle exec minitest-queue ./test/samples/does_not_exist.rb
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed"
+}
+
+@test "minitest-queue fails when given a malformed test file" {
+  [ -f README.md ]
+  run bundle exec minitest-queue README.md
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed"
+}
+
+@test "minitest-queue handles test file being deleted" {
+  cp test/samples/sample_mini{test5,spec}.rb $SCRATCH
+
+  run bundle exec minitest-queue $SCRATCH/*
+  assert_status 0
+  assert_output_contains "Meme::when asked about blending possibilities"
+
+  rm $SCRATCH/sample_minispec.rb
+
+  run bundle exec minitest-queue $SCRATCH/*
+  assert_status 0
+  refute_output_contains "Meme::when asked about blending possibilities"
+}
+
+@test "minitest-queue handles suites changing inside a file" {
+  cp test/samples/sample_minispec.rb $SCRATCH
+
+  run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
+  assert_status 0
+  assert_output_contains "Meme::when asked about blending possibilities"
+
+  sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/sample_minispec.rb
+
+  run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
+  assert_status 0
+  assert_output_contains "Meme2::when asked about blending possibilities"
+}

data/test/rspec.bats

@@ -0,0 +1,38 @@
+load "testlib"
+
+setup() {
+  require_gem "rspec" ">= 2.0"
+}
+
+@test "rspec-queue succeeds when all specs pass" {
+  run bundle exec rspec-queue ./test/samples/sample_spec.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "rspec-queue fails when a spec fails" {
+  export FAIL=1
+  run bundle exec rspec-queue ./test/samples/sample_spec.rb
+  assert_status 1
+  assert_output_contains "1) RSpecFailure fails"
+  assert_output_contains "Failure/Error: expect(:foo).to eq :bar"
+}
+
+@test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" {
+  export TEST_QUEUE_SPLIT_GROUPS=true
+  run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
+  assert_status 0
+
+  assert_output_matches '\[ 1\] +1 example, 0 failures'
+  assert_output_matches '\[ 2\] +1 example, 0 failures'
+}
+
+@test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" {
+  export TEST_QUEUE_SPLIT_GROUPS=true
+  export NOSPLIT=1
+  run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
+  assert_status 0
+
+  assert_output_contains "2 examples, 0 failures"
+  assert_output_contains "0 examples, 0 failures"
+}

data/{features → test/samples/features}/step_definitions/common.rb

@@ -7,7 +7,11 @@ When(/^b$/) do
 end
 
 When(/^bad$/) do
-  1.should == 0
+  if ENV["FAIL"]
+    1.should == 0
+  else
+    1.should == 1
+  end
 end
 
 Then(/^c$/) do

data/test/{sample_minispec.rb → samples/sample_minispec.rb}

@@ -27,5 +27,11 @@ describe Meme do
       sleep 0.1
       @meme.will_it_blend?.wont_match /^no/i
     end
+
+    if ENV["FAIL"]
+      it "fails" do
+        @meme.will_it_blend?.must_equal "NO!"
+      end
+    end
   end
 end

data/test/{sample_minitest4.rb → samples/sample_minitest4.rb}

@@ -16,8 +16,10 @@ end
   end)
 end
 
-class MiniTestFailure < MiniTest::Unit::TestCase
-  def test_fail
-    assert_equal 0, 1
+if ENV["FAIL"]
+  class MiniTestFailure < MiniTest::Unit::TestCase
+    def test_fail
+      assert_equal 0, 1
+    end
   end
 end

data/test/{sample_minitest5.rb → samples/sample_minitest5.rb}

@@ -16,8 +16,10 @@ end
   end)
 end
 
-class MiniTestFailure < MiniTest::Test
-  def test_fail
-    assert_equal 0, 1
+if ENV["FAIL"]
+  class MiniTestFailure < MiniTest::Test
+    def test_fail
+      assert_equal 0, 1
+    end
   end
 end

data/test/{sample_spec.rb → samples/sample_spec.rb}

@@ -16,8 +16,10 @@ end
   end
 end
 
-describe 'RSpecFailure' do
-  it 'fails' do
-    expect(:foo).to eq :bar
+if ENV["FAIL"]
+  describe 'RSpecFailure' do
+    it 'fails' do
+      expect(:foo).to eq :bar
+    end
   end
 end