test-queue-patched 0.4.3

data/lib/test_queue/test_framework.rb

@@ -0,0 +1,29 @@
+module TestQueue
+  # This class provides an abstraction over the various test frameworks we
+  # support. The framework-specific subclasses are defined in the various
+  # test_queue/runner/* files.
+  class TestFramework
+    # Return all file paths to load test suites from.
+    #
+    # An example implementation might just return files passed on the command
+    # line, or defer to the underlying test framework to determine which files
+    # to load.
+    #
+    # Returns an Enumerable of String file paths.
+    def all_suite_files
+      raise NotImplementedError
+    end
+
+    # Load all suites from the specified file path.
+    #
+    # path - String file path to load suites from
+    #
+    # Returns an Enumerable of tuples containing:
+    #   suite_name   - String that uniquely identifies this suite
+    #   suite        - Framework-specific object that can be used to actually
+    #                  run the suite
+    def suites_from_file(path)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/test_queue.rb

@@ -0,0 +1,8 @@
+if !IO.respond_to?(:binread)
+  class << IO
+    alias :binread :read
+  end
+end
+
+require 'test_queue/iterator'
+require 'test_queue/runner'

data/script/bootstrap

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -ex
+
+cd "$(dirname "$0")/.."
+ROOT=$(pwd)
+
+rm -rf tmp
+mkdir tmp
+mkdir -p vendor/bats
+curl --silent --location --show-error https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz | tar -xz -C tmp
+tmp/bats-0.4.0/install.sh "$ROOT/vendor/bats"

data/script/cibuild

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -ex
+
+if [ "$SUITE" = "ruby" ]; then
+  exec script/spec
+fi
+
+export TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1
+
+if [ -f "Gemfile-$SUITE" ]; then
+  export BUNDLE_GEMFILE="Gemfile-$SUITE"
+else
+  export BUNDLE_GEMFILE=Gemfile
+fi
+
+bundle install
+
+vendor/bats/bin/bats test

data/script/spec

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+export BUNDLE_GEMFILE=Gemfile-rspec3-2
+bundle install
+bundle exec rspec spec

data/spec/stats_spec.rb

@@ -0,0 +1,76 @@
+require "fileutils"
+require "tempfile"
+require "test_queue/stats"
+
+RSpec.describe TestQueue::Stats do
+  before do
+    Tempfile.open("test_queue_stats") do |f|
+      @path = f.path
+      f.close!
+    end
+  end
+
+  after do
+    FileUtils.rm_f(@path)
+  end
+
+  describe "#initialize" do
+    it "ignores empty stats files" do
+      File.write(@path, "")
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores invalid data in the stats files" do
+      File.write(@path, "this is not marshal data")
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores badly-typed data in the stats file" do
+      File.write(@path, Marshal.dump(["heyyy"]))
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+
+    it "ignores stats files with a wrong version number" do
+      File.write(@path, Marshal.dump({ :version => 1e8, :suites => "boom" }))
+      stats = TestQueue::Stats.new(@path)
+      expect(stats.all_suites).to be_empty
+    end
+  end
+
+  it "can save and load data" do
+    stats = TestQueue::Stats.new(@path)
+    time = truncated_now
+    suites = [
+      TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
+      TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time + 5),
+    ]
+    stats.record_suites(suites)
+    stats.save
+
+    stats = TestQueue::Stats.new(@path)
+    expect(stats.all_suites.sort_by(&:name)).to eq(suites)
+  end
+
+  it "prunes suites not seen in the last 8 days" do
+    stats = TestQueue::Stats.new(@path)
+    time = truncated_now
+    suites = [
+      TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
+      TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time - (8 * 24 * 60 * 60) - 2),
+      TestQueue::Stats::Suite.new("Suite3", "baz.rb", 0.6, time - (7 * 24 * 60 * 60)),
+    ]
+    stats.record_suites(suites)
+    stats.save
+
+    stats = TestQueue::Stats.new(@path)
+    expect(stats.all_suites.map(&:name).sort).to eq(%w[Suite1 Suite3])
+  end
+
+  # Returns Time.now rounded down to the nearest second.
+  def truncated_now
+    Time.at(Time.now.to_i)
+  end
+end

data/test/cucumber.bats

@@ -0,0 +1,57 @@
+load "testlib"
+
+SCRATCH=tmp/cucumber-tests
+
+setup() {
+  require_gem "cucumber" ">= 1.0"
+  rm -rf $SCRATCH
+  mkdir -p $SCRATCH
+}
+
+teardown() {
+  rm -rf $SCRATCH
+}
+
+@test "cucumber-queue succeeds when all features pass" {
+  run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "cucumber-queue fails when a feature fails" {
+  export FAIL=1
+  run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
+  assert_status 2
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "cucumber test/samples/features/bad.feature:2 # Scenario: failure"
+  assert_output_contains "cucumber test/samples/features/sample2.feature:26 # Scenario: failure"
+}
+
+@test "cucumber-queue fails when given a missing feature" {
+  run bundle exec cucumber-queue test/samples/does_not_exist.feature --require test/samples/features/step_definitions
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed."
+}
+
+@test "cucumber-queue fails when given a malformed feature" {
+  [ -f README.md ]
+  run bundle exec cucumber-queue README.md --require test/samples/features/step_definitions
+
+  # Cucumber 1 and 2 fail in different ways.
+  refute_status 0
+  assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:'
+}
+
+@test "cucumber-queue handles test file being deleted" {
+  cp test/samples/features/*.feature $SCRATCH
+
+  run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
+  assert_status 0
+  assert_output_matches "Feature: Foobar$"
+
+  rm $SCRATCH/sample.feature
+
+  run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
+  assert_status 0
+  refute_output_matches "Feature: Foobar$"
+}

data/test/minitest4.bats

@@ -0,0 +1,34 @@
+load "testlib"
+
+setup() {
+  require_gem "minitest" ">= 4.0"
+}
+
+@test "minitest-queue succeeds when all tests pass" {
+  run bundle exec minitest-queue ./test/samples/*_minitest4.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue fails when a test fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minitest4.rb
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "minitest-queue succeeds when all specs pass" {
+  run bundle exec minitest-queue ./test/samples/*_minispec.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue fails when a spec fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minispec.rb
+  assert_status 1
+  assert_output_contains "1) Failure:"
+  assert_output_contains "Meme::when asked about blending possibilities#test_0002_fails"
+}

data/test/minitest5.bats

@@ -0,0 +1,194 @@
+load "testlib"
+
+SCRATCH=tmp/minitest5-tests
+
+setup() {
+  require_gem "minitest" ">= 5.0"
+  rm -rf $SCRATCH
+  mkdir -p $SCRATCH
+}
+
+teardown() {
+  rm -rf $SCRATCH
+}
+
+@test "minitest-queue (minitest5) succeeds when all tests pass" {
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "minitest-queue (minitest5) fails when a test fails" {
+  export FAIL=1
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "TEST_QUEUE_FORCE whitelists certain tests" {
+  export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,MiniTestSleep8"
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "MiniTestSleep21"
+  assert_output_contains "MiniTestSleep8"
+  refute_output_contains "MiniTestSleep9"
+}
+
+assert_test_queue_force_ordering() {
+  run bundle exec minitest-queue "$@"
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+
+  # Turn the list of suites that were run into a comma-separated list. Input
+  # looks like:
+  #     SuiteName: .  <0.001>
+  actual_tests=$(echo "$output" | \
+                 egrep '^    .*: \.+  <' | \
+                 sed -E -e 's/^    (.*): \.+.*/\1/' | \
+                 tr '\n' ',' | \
+                 sed -e 's/,$//')
+  assert_equal "$TEST_QUEUE_FORCE" "$actual_tests"
+}
+
+@test "TEST_QUEUE_FORCE ensures test ordering" {
+  export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="Meme::when asked about cheeseburgers,MiniTestEqual"
+
+  # Without stats file
+  rm -f .test_queue_stats
+  assert_test_queue_force_ordering ./test/samples/sample_minitest5.rb ./test/samples/sample_minispec.rb
+  rm -f .test_queue_stats
+  assert_test_queue_force_ordering ./test/samples/sample_minispec.rb ./test/samples/sample_minitest5.rb
+
+  # With stats file
+  assert_test_queue_force_ordering ./test/samples/sample_minitest5.rb ./test/samples/sample_minispec.rb
+  assert_test_queue_force_ordering ./test/samples/sample_minispec.rb ./test/samples/sample_minitest5.rb
+}
+
+@test "minitest-queue fails if TEST_QUEUE_FORCE specifies nonexistent tests" {
+  export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,DoesNotExist"
+  run bundle exec minitest-queue ./test/samples/*_minitest5.rb
+  assert_status 1
+  assert_output_contains "Failed to discover DoesNotExist specified in TEST_QUEUE_FORCE"
+}
+
+@test "multi-master central master succeeds when all tests pass" {
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  export SLEEP_AS_RELAY=1
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "multi-master remote master succeeds when all tests pass" {
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  export SLEEP_AS_MASTER=1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "multi-master central master fails when a test fails" {
+  export FAIL=1
+  export SLEEP_AS_RELAY=1
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "multi-master remote master fails when a test fails" {
+  export FAIL=1
+  export SLEEP_AS_MASTER=1
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb || true &
+  sleep 0.1
+  TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 1
+  assert_output_contains "Starting test-queue master"
+  assert_output_contains "1) Failure:"
+  assert_output_contains "MiniTestFailure#test_fail"
+}
+
+@test "multi-master central master prints out remote master messages" {
+  export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
+  TEST_QUEUE_RELAY=0.0.0.0:12345 TEST_QUEUE_REMOTE_MASTER_MESSAGE="hello from remote master" bundle exec minitest-queue ./test/samples/sample_minitest5.rb &
+  TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  wait
+
+  assert_status 0
+  assert_output_contains "hello from remote master"
+}
+
+@test "recovers from child processes dying in an unorderly way" {
+  export KILL=1
+  run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  assert_status 1
+  assert_output_contains "SIGKILL (signal 9)"
+}
+
+@test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" {
+  export TEST_QUEUE_WORKERS=0
+  run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
+  assert_status 1
+  assert_output_contains "Worker count (0) must be greater than 0"
+}
+
+@test "minitest-queue fails when given a missing test file" {
+  run bundle exec minitest-queue ./test/samples/does_not_exist.rb
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed"
+}
+
+@test "minitest-queue fails when given a malformed test file" {
+  [ -f README.md ]
+  run bundle exec minitest-queue README.md
+  assert_status 1
+  assert_output_contains "Aborting: Discovering suites failed"
+}
+
+@test "minitest-queue handles test file being deleted" {
+  cp test/samples/sample_mini{test5,spec}.rb $SCRATCH
+
+  run bundle exec minitest-queue $SCRATCH/*
+  assert_status 0
+  assert_output_contains "Meme::when asked about blending possibilities"
+
+  rm $SCRATCH/sample_minispec.rb
+
+  run bundle exec minitest-queue $SCRATCH/*
+  assert_status 0
+  refute_output_contains "Meme::when asked about blending possibilities"
+}
+
+@test "minitest-queue handles suites changing inside a file" {
+  cp test/samples/sample_minispec.rb $SCRATCH
+
+  run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
+  assert_status 0
+  assert_output_contains "Meme::when asked about blending possibilities"
+
+  sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/sample_minispec.rb
+
+  run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
+  assert_status 0
+  assert_output_contains "Meme2::when asked about blending possibilities"
+}

data/test/rspec.bats

@@ -0,0 +1,46 @@
+load "testlib"
+
+setup() {
+  require_gem "rspec" ">= 2.0"
+}
+
+@test "rspec-queue succeeds when all specs pass" {
+  run bundle exec rspec-queue ./test/samples/sample_spec.rb
+  assert_status 0
+  assert_output_contains "Starting test-queue master"
+}
+
+@test "rspec-queue fails when a spec fails" {
+  export FAIL=1
+  run bundle exec rspec-queue ./test/samples/sample_spec.rb
+  assert_status 1
+  assert_output_contains "1) RSpecFailure fails"
+  assert_output_contains "Failure/Error: expect(:foo).to eq :bar"
+}
+
+@test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" {
+  export TEST_QUEUE_SPLIT_GROUPS=true
+  run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
+  assert_status 0
+
+  assert_output_matches '\[ 1\] +1 example, 0 failures'
+  assert_output_matches '\[ 2\] +1 example, 0 failures'
+}
+
+@test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" {
+  export TEST_QUEUE_SPLIT_GROUPS=true
+  export NOSPLIT=1
+  run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
+  assert_status 0
+
+  assert_output_contains "2 examples, 0 failures"
+  assert_output_contains "0 examples, 0 failures"
+}
+
+@test "rspec-queue supports shared example groups" {
+  run bundle exec rspec-queue ./test/samples/sample_use_shared_example1_spec.rb \
+                              ./test/samples/sample_use_shared_example2_spec.rb
+  assert_status 0
+
+}
+

data/test/samples/features/bad.feature

@@ -0,0 +1,5 @@
+Feature: bad
+  Scenario: failure
+    Given a
+    When bad
+    Then c

data/test/samples/features/sample.feature

@@ -0,0 +1,25 @@
+Feature: Foobar
+  Scenario: Bar
+    Given a
+    When b
+    Then c
+  Scenario: Baz
+    Given a
+    When b
+    Then c
+  Scenario: Bam
+    Given a
+    When b
+    Then c
+  Scenario: Car
+    Given a
+    When b
+    Then c
+  Scenario: Caz
+    Given a
+    When b
+    Then c
+  Scenario: Cam
+    Given a
+    When b
+    Then c

data/test/samples/features/sample2.feature

@@ -0,0 +1,29 @@
+Feature: Foobar2
+  Scenario: Bar
+    Given a
+    When b
+    Then c
+  Scenario: Baz
+    Given a
+    When b
+    Then c
+  Scenario: Bam
+    Given a
+    When b
+    Then c
+  Scenario: Car
+    Given a
+    When b
+    Then c
+  Scenario: Caz
+    Given a
+    When b
+    Then c
+  Scenario: Cam
+    Given a
+    When b
+    Then c
+  Scenario: failure
+    Given a
+    When bad
+    Then c

data/test/samples/features/step_definitions/common.rb

@@ -0,0 +1,19 @@
+Given(/^a$/) do
+  sleep 0.10
+end
+
+When(/^b$/) do
+  sleep 0.25
+end
+
+When(/^bad$/) do
+  if ENV["FAIL"]
+    1.should == 0
+  else
+    1.should == 1
+  end
+end
+
+Then(/^c$/) do
+  1.should == 1
+end

data/test/samples/sample_minispec.rb

@@ -0,0 +1,37 @@
+require 'minitest/spec'
+
+class Meme
+  def i_can_has_cheezburger?
+    "OHAI!"
+  end
+
+  def will_it_blend?
+    "YES!"
+  end
+end
+
+describe Meme do
+  before do
+    @meme = Meme.new
+  end
+
+  describe "when asked about cheeseburgers" do
+    it "must respond positively" do
+      sleep 0.1
+      @meme.i_can_has_cheezburger?.must_equal "OHAI!"
+    end
+  end
+
+  describe "when asked about blending possibilities" do
+    it "won't say no" do
+      sleep 0.1
+      @meme.will_it_blend?.wont_match /^no/i
+    end
+
+    if ENV["FAIL"]
+      it "fails" do
+        @meme.will_it_blend?.must_equal "NO!"
+      end
+    end
+  end
+end

data/test/samples/sample_minitest4.rb

@@ -0,0 +1,25 @@
+require 'minitest/unit'
+
+class MiniTestEqual < MiniTest::Unit::TestCase
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+30.times do |i|
+  Object.const_set("MiniTestSleep#{i}", Class.new(MiniTest::Unit::TestCase) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
+end
+
+if ENV["FAIL"]
+  class MiniTestFailure < MiniTest::Unit::TestCase
+    def test_fail
+      assert_equal 0, 1
+    end
+  end
+end

data/test/samples/sample_minitest5.rb

@@ -0,0 +1,33 @@
+require 'minitest/autorun'
+
+class MiniTestEqual < MiniTest::Test
+  def test_equal
+    assert_equal 1, 1
+  end
+end
+
+30.times do |i|
+  Object.const_set("MiniTestSleep#{i}", Class.new(MiniTest::Test) do
+    define_method('test_sleep') do
+      start = Time.now
+      sleep(0.25)
+      assert_in_delta Time.now-start, 0.25, 0.02
+    end
+  end)
+end
+
+if ENV["FAIL"]
+  class MiniTestFailure < MiniTest::Test
+    def test_fail
+      assert_equal 0, 1
+    end
+  end
+end
+
+if ENV["KILL"]
+  class MiniTestKilledFailure < MiniTest::Test
+    def test_kill
+      Process.kill(9, $$)
+    end
+  end
+end

data/test/samples/sample_rspec_helper.rb

@@ -0,0 +1 @@
+require_relative 'sample_shared_examples_for_spec'

data/test/samples/sample_shared_examples_for_spec.rb

@@ -0,0 +1,5 @@
+
+shared_examples_for 'Sample Example' do
+  it { should eq 5 }
+end
+