test-queue 0.2.13 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.travis.yml +18 -0
  4. data/Gemfile +2 -0
  5. data/Gemfile-cucumber1-3 +4 -0
  6. data/Gemfile-cucumber1-3.lock +33 -0
  7. data/Gemfile-cucumber2-4 +4 -0
  8. data/Gemfile-cucumber2-4.lock +37 -0
  9. data/Gemfile-minitest4.lock +4 -23
  10. data/Gemfile-minitest5 +3 -0
  11. data/Gemfile-minitest5.lock +19 -0
  12. data/Gemfile-rspec2-1 +3 -0
  13. data/Gemfile-rspec2-1.lock +27 -0
  14. data/Gemfile-rspec3-0.lock +4 -13
  15. data/Gemfile-rspec3-1.lock +4 -13
  16. data/Gemfile-rspec3-2.lock +4 -13
  17. data/Gemfile-testunit.lock +4 -27
  18. data/Gemfile.lock +4 -1
  19. data/README.md +4 -0
  20. data/bin/minitest-queue +0 -1
  21. data/bin/testunit-queue +0 -1
  22. data/lib/test_queue/iterator.rb +41 -10
  23. data/lib/test_queue/runner.rb +167 -58
  24. data/lib/test_queue/runner/cucumber.rb +81 -12
  25. data/lib/test_queue/runner/minitest.rb +0 -4
  26. data/lib/test_queue/runner/minitest4.rb +25 -2
  27. data/lib/test_queue/runner/minitest5.rb +36 -11
  28. data/lib/test_queue/runner/rspec.rb +55 -7
  29. data/lib/test_queue/runner/rspec2.rb +11 -8
  30. data/lib/test_queue/runner/rspec3.rb +10 -7
  31. data/lib/test_queue/runner/sample.rb +0 -2
  32. data/lib/test_queue/runner/testunit.rb +25 -7
  33. data/lib/test_queue/stats.rb +95 -0
  34. data/lib/test_queue/test_framework.rb +29 -0
  35. data/script/bootstrap +12 -0
  36. data/script/cibuild +19 -0
  37. data/script/spec +7 -0
  38. data/spec/stats_spec.rb +76 -0
  39. data/test-queue.gemspec +1 -4
  40. data/test/cucumber.bats +57 -0
  41. data/test/minitest4.bats +34 -0
  42. data/test/minitest5.bats +111 -0
  43. data/test/rspec.bats +38 -0
  44. data/{features → test/samples/features}/bad.feature +0 -0
  45. data/{features → test/samples/features}/sample.feature +0 -0
  46. data/{features → test/samples/features}/sample2.feature +0 -0
  47. data/{features → test/samples/features}/step_definitions/common.rb +5 -1
  48. data/test/{sample_minispec.rb → samples/sample_minispec.rb} +6 -0
  49. data/test/{sample_minitest4.rb → samples/sample_minitest4.rb} +5 -3
  50. data/test/{sample_minitest5.rb → samples/sample_minitest5.rb} +5 -3
  51. data/test/{sample_spec.rb → samples/sample_spec.rb} +5 -3
  52. data/test/samples/sample_split_spec.rb +17 -0
  53. data/test/{sample_testunit.rb → samples/sample_testunit.rb} +5 -3
  54. data/test/testlib.bash +81 -0
  55. data/test/testunit.bats +20 -0
  56. metadata +40 -60
  57. data/test-multi.sh +0 -8
  58. data/test.sh +0 -23
@@ -0,0 +1,12 @@
1
+ #!/bin/sh
2
+
3
+ set -ex
4
+
5
+ cd "$(dirname "$0")/.."
6
+ ROOT=$(pwd)
7
+
8
+ rm -rf tmp
9
+ mkdir tmp
10
+ mkdir -p vendor/bats
11
+ curl --silent --location --show-error https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz | tar -xz -C tmp
12
+ tmp/bats-0.4.0/install.sh "$ROOT/vendor/bats"
@@ -0,0 +1,19 @@
1
+ #!/bin/sh
2
+
3
+ set -ex
4
+
5
+ if [ "$SUITE" = "ruby" ]; then
6
+ exec script/spec
7
+ fi
8
+
9
+ export TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1
10
+
11
+ if [ -f "Gemfile-$SUITE" ]; then
12
+ export BUNDLE_GEMFILE="Gemfile-$SUITE"
13
+ else
14
+ export BUNDLE_GEMFILE=Gemfile
15
+ fi
16
+
17
+ bundle install
18
+
19
+ vendor/bats/bin/bats test
@@ -0,0 +1,7 @@
1
+ #!/bin/sh
2
+
3
+ set -ex
4
+
5
+ export BUNDLE_GEMFILE=Gemfile-rspec3-2
6
+ bundle install
7
+ bundle exec rspec spec
@@ -0,0 +1,76 @@
1
+ require "fileutils"
2
+ require "tempfile"
3
+ require "test_queue/stats"
4
+
5
+ RSpec.describe TestQueue::Stats do
6
+ before do
7
+ Tempfile.open("test_queue_stats") do |f|
8
+ @path = f.path
9
+ f.close!
10
+ end
11
+ end
12
+
13
+ after do
14
+ FileUtils.rm_f(@path)
15
+ end
16
+
17
+ describe "#initialize" do
18
+ it "ignores empty stats files" do
19
+ File.write(@path, "")
20
+ stats = TestQueue::Stats.new(@path)
21
+ expect(stats.all_suites).to be_empty
22
+ end
23
+
24
+ it "ignores invalid data in the stats files" do
25
+ File.write(@path, "this is not marshal data")
26
+ stats = TestQueue::Stats.new(@path)
27
+ expect(stats.all_suites).to be_empty
28
+ end
29
+
30
+ it "ignores badly-typed data in the stats file" do
31
+ File.write(@path, Marshal.dump(["heyyy"]))
32
+ stats = TestQueue::Stats.new(@path)
33
+ expect(stats.all_suites).to be_empty
34
+ end
35
+
36
+ it "ignores stats files with a wrong version number" do
37
+ File.write(@path, Marshal.dump({ :version => 1e8, :suites => "boom" }))
38
+ stats = TestQueue::Stats.new(@path)
39
+ expect(stats.all_suites).to be_empty
40
+ end
41
+ end
42
+
43
+ it "can save and load data" do
44
+ stats = TestQueue::Stats.new(@path)
45
+ time = truncated_now
46
+ suites = [
47
+ TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
48
+ TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time + 5),
49
+ ]
50
+ stats.record_suites(suites)
51
+ stats.save
52
+
53
+ stats = TestQueue::Stats.new(@path)
54
+ expect(stats.all_suites.sort_by(&:name)).to eq(suites)
55
+ end
56
+
57
+ it "prunes suites not seen in the last 8 days" do
58
+ stats = TestQueue::Stats.new(@path)
59
+ time = truncated_now
60
+ suites = [
61
+ TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
62
+ TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time - (8 * 24 * 60 * 60) - 2),
63
+ TestQueue::Stats::Suite.new("Suite3", "baz.rb", 0.6, time - (7 * 24 * 60 * 60)),
64
+ ]
65
+ stats.record_suites(suites)
66
+ stats.save
67
+
68
+ stats = TestQueue::Stats.new(@path)
69
+ expect(stats.all_suites.map(&:name).sort).to eq(%w[Suite1 Suite3])
70
+ end
71
+
72
+ # Returns Time.now rounded down to the nearest second.
73
+ def truncated_now
74
+ Time.at(Time.now.to_i)
75
+ end
76
+ end
@@ -1,6 +1,6 @@
1
1
  spec = Gem::Specification.new do |s|
2
2
  s.name = 'test-queue'
3
- s.version = '0.2.13'
3
+ s.version = '0.3.0'
4
4
  s.summary = 'parallel test runner'
5
5
  s.description = 'minitest/rspec parallel test runner for CI environments'
6
6
 
@@ -17,8 +17,5 @@ spec = Gem::Specification.new do |s|
17
17
  s.executables << 'testunit-queue'
18
18
  s.executables << 'cucumber-queue'
19
19
 
20
- s.add_development_dependency 'rspec', '>= 2.13', '< 4.0'
21
- s.add_development_dependency 'cucumber', '~> 1.3.10'
22
-
23
20
  s.files = `git ls-files`.split("\n")
24
21
  end
@@ -0,0 +1,57 @@
1
+ load "testlib"
2
+
3
+ SCRATCH=tmp/cucumber-tests
4
+
5
+ setup() {
6
+ require_gem "cucumber" ">= 1.0"
7
+ rm -rf $SCRATCH
8
+ mkdir -p $SCRATCH
9
+ }
10
+
11
+ teardown() {
12
+ rm -rf $SCRATCH
13
+ }
14
+
15
+ @test "cucumber-queue succeeds when all features pass" {
16
+ run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
17
+ assert_status 0
18
+ assert_output_contains "Starting test-queue master"
19
+ }
20
+
21
+ @test "cucumber-queue fails when a feature fails" {
22
+ export FAIL=1
23
+ run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
24
+ assert_status 2
25
+ assert_output_contains "Starting test-queue master"
26
+ assert_output_contains "cucumber test/samples/features/bad.feature:2 # Scenario: failure"
27
+ assert_output_contains "cucumber test/samples/features/sample2.feature:26 # Scenario: failure"
28
+ }
29
+
30
+ @test "cucumber-queue fails when given a missing feature" {
31
+ run bundle exec cucumber-queue test/samples/does_not_exist.feature --require test/samples/features/step_definitions
32
+ assert_status 1
33
+ assert_output_contains "Aborting: Discovering suites failed."
34
+ }
35
+
36
+ @test "cucumber-queue fails when given a malformed feature" {
37
+ [ -f README.md ]
38
+ run bundle exec cucumber-queue README.md --require test/samples/features/step_definitions
39
+
40
+ # Cucumber 1 and 2 fail in different ways.
41
+ refute_status 0
42
+ assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:'
43
+ }
44
+
45
+ @test "cucumber-queue handles test file being deleted" {
46
+ cp test/samples/features/*.feature $SCRATCH
47
+
48
+ run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
49
+ assert_status 0
50
+ assert_output_matches "Feature: Foobar$"
51
+
52
+ rm $SCRATCH/sample.feature
53
+
54
+ run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
55
+ assert_status 0
56
+ refute_output_matches "Feature: Foobar$"
57
+ }
@@ -0,0 +1,34 @@
1
+ load "testlib"
2
+
3
+ setup() {
4
+ require_gem "minitest" ">= 4.0"
5
+ }
6
+
7
+ @test "minitest-queue succeeds when all tests pass" {
8
+ run bundle exec minitest-queue ./test/samples/*_minitest4.rb
9
+ assert_status 0
10
+ assert_output_contains "Starting test-queue master"
11
+ }
12
+
13
+ @test "minitest-queue fails when a test fails" {
14
+ export FAIL=1
15
+ run bundle exec minitest-queue ./test/samples/*_minitest4.rb
16
+ assert_status 1
17
+ assert_output_contains "Starting test-queue master"
18
+ assert_output_contains "1) Failure:"
19
+ assert_output_contains "MiniTestFailure#test_fail"
20
+ }
21
+
22
+ @test "minitest-queue succeeds when all specs pass" {
23
+ run bundle exec minitest-queue ./test/samples/*_minispec.rb
24
+ assert_status 0
25
+ assert_output_contains "Starting test-queue master"
26
+ }
27
+
28
+ @test "minitest-queue fails when a spec fails" {
29
+ export FAIL=1
30
+ run bundle exec minitest-queue ./test/samples/*_minispec.rb
31
+ assert_status 1
32
+ assert_output_contains "1) Failure:"
33
+ assert_output_contains "Meme::when asked about blending possibilities#test_0002_fails"
34
+ }
@@ -0,0 +1,111 @@
1
+ load "testlib"
2
+
3
+ SCRATCH=tmp/minitest5-tests
4
+
5
+ setup() {
6
+ require_gem "minitest" ">= 5.0"
7
+ rm -rf $SCRATCH
8
+ mkdir -p $SCRATCH
9
+ }
10
+
11
+ teardown() {
12
+ rm -rf $SCRATCH
13
+ }
14
+
15
+ @test "minitest-queue (minitest5) succeeds when all tests pass" {
16
+ run bundle exec minitest-queue ./test/samples/*_minitest5.rb
17
+ assert_status 0
18
+ assert_output_contains "Starting test-queue master"
19
+ }
20
+
21
+ @test "minitest-queue (minitest5) fails when a test fails" {
22
+ export FAIL=1
23
+ run bundle exec minitest-queue ./test/samples/*_minitest5.rb
24
+ assert_status 1
25
+ assert_output_contains "Starting test-queue master"
26
+ assert_output_contains "1) Failure:"
27
+ assert_output_contains "MiniTestFailure#test_fail"
28
+ }
29
+
30
+ @test "TEST_QUEUE_FORCE whitelists certain tests" {
31
+ export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,MiniTestSleep8"
32
+ run bundle exec minitest-queue ./test/samples/*_minitest5.rb
33
+ assert_status 0
34
+ assert_output_contains "Starting test-queue master"
35
+ assert_output_contains "MiniTestSleep21"
36
+ assert_output_contains "MiniTestSleep8"
37
+ refute_output_contains "MiniTestSleep9"
38
+ }
39
+
40
+ @test "multi-master succeeds when all tests pass" {
41
+ export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
42
+ TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
43
+ sleep 0.1
44
+ TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
45
+ wait
46
+
47
+ assert_status 0
48
+ assert_output_contains "Starting test-queue master"
49
+ }
50
+
51
+ @test "multi-master fails when a test fails" {
52
+ export FAIL=1
53
+ export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
54
+ TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
55
+ sleep 0.1
56
+ TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
57
+ wait
58
+
59
+ assert_status 1
60
+ assert_output_contains "Starting test-queue master"
61
+ assert_output_contains "1) Failure:"
62
+ assert_output_contains "MiniTestFailure#test_fail"
63
+ }
64
+
65
+ @test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" {
66
+ export TEST_QUEUE_WORKERS=0
67
+ run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
68
+ assert_status 1
69
+ assert_output_contains "Worker count (0) must be greater than 0"
70
+ }
71
+
72
+ @test "minitest-queue fails when given a missing test file" {
73
+ run bundle exec minitest-queue ./test/samples/does_not_exist.rb
74
+ assert_status 1
75
+ assert_output_contains "Aborting: Discovering suites failed"
76
+ }
77
+
78
+ @test "minitest-queue fails when given a malformed test file" {
79
+ [ -f README.md ]
80
+ run bundle exec minitest-queue README.md
81
+ assert_status 1
82
+ assert_output_contains "Aborting: Discovering suites failed"
83
+ }
84
+
85
+ @test "minitest-queue handles test file being deleted" {
86
+ cp test/samples/sample_mini{test5,spec}.rb $SCRATCH
87
+
88
+ run bundle exec minitest-queue $SCRATCH/*
89
+ assert_status 0
90
+ assert_output_contains "Meme::when asked about blending possibilities"
91
+
92
+ rm $SCRATCH/sample_minispec.rb
93
+
94
+ run bundle exec minitest-queue $SCRATCH/*
95
+ assert_status 0
96
+ refute_output_contains "Meme::when asked about blending possibilities"
97
+ }
98
+
99
+ @test "minitest-queue handles suites changing inside a file" {
100
+ cp test/samples/sample_minispec.rb $SCRATCH
101
+
102
+ run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
103
+ assert_status 0
104
+ assert_output_contains "Meme::when asked about blending possibilities"
105
+
106
+ sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/sample_minispec.rb
107
+
108
+ run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
109
+ assert_status 0
110
+ assert_output_contains "Meme2::when asked about blending possibilities"
111
+ }
@@ -0,0 +1,38 @@
1
+ load "testlib"
2
+
3
+ setup() {
4
+ require_gem "rspec" ">= 2.0"
5
+ }
6
+
7
+ @test "rspec-queue succeeds when all specs pass" {
8
+ run bundle exec rspec-queue ./test/samples/sample_spec.rb
9
+ assert_status 0
10
+ assert_output_contains "Starting test-queue master"
11
+ }
12
+
13
+ @test "rspec-queue fails when a spec fails" {
14
+ export FAIL=1
15
+ run bundle exec rspec-queue ./test/samples/sample_spec.rb
16
+ assert_status 1
17
+ assert_output_contains "1) RSpecFailure fails"
18
+ assert_output_contains "Failure/Error: expect(:foo).to eq :bar"
19
+ }
20
+
21
+ @test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" {
22
+ export TEST_QUEUE_SPLIT_GROUPS=true
23
+ run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
24
+ assert_status 0
25
+
26
+ assert_output_matches '\[ 1\] +1 example, 0 failures'
27
+ assert_output_matches '\[ 2\] +1 example, 0 failures'
28
+ }
29
+
30
+ @test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" {
31
+ export TEST_QUEUE_SPLIT_GROUPS=true
32
+ export NOSPLIT=1
33
+ run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
34
+ assert_status 0
35
+
36
+ assert_output_contains "2 examples, 0 failures"
37
+ assert_output_contains "0 examples, 0 failures"
38
+ }
@@ -7,7 +7,11 @@ When(/^b$/) do
7
7
  end
8
8
 
9
9
  When(/^bad$/) do
10
- 1.should == 0
10
+ if ENV["FAIL"]
11
+ 1.should == 0
12
+ else
13
+ 1.should == 1
14
+ end
11
15
  end
12
16
 
13
17
  Then(/^c$/) do
@@ -27,5 +27,11 @@ describe Meme do
27
27
  sleep 0.1
28
28
  @meme.will_it_blend?.wont_match /^no/i
29
29
  end
30
+
31
+ if ENV["FAIL"]
32
+ it "fails" do
33
+ @meme.will_it_blend?.must_equal "NO!"
34
+ end
35
+ end
30
36
  end
31
37
  end
@@ -16,8 +16,10 @@ end
16
16
  end)
17
17
  end
18
18
 
19
- class MiniTestFailure < MiniTest::Unit::TestCase
20
- def test_fail
21
- assert_equal 0, 1
19
+ if ENV["FAIL"]
20
+ class MiniTestFailure < MiniTest::Unit::TestCase
21
+ def test_fail
22
+ assert_equal 0, 1
23
+ end
22
24
  end
23
25
  end
@@ -16,8 +16,10 @@ end
16
16
  end)
17
17
  end
18
18
 
19
- class MiniTestFailure < MiniTest::Test
20
- def test_fail
21
- assert_equal 0, 1
19
+ if ENV["FAIL"]
20
+ class MiniTestFailure < MiniTest::Test
21
+ def test_fail
22
+ assert_equal 0, 1
23
+ end
22
24
  end
23
25
  end
@@ -16,8 +16,10 @@ end
16
16
  end
17
17
  end
18
18
 
19
- describe 'RSpecFailure' do
20
- it 'fails' do
21
- expect(:foo).to eq :bar
19
+ if ENV["FAIL"]
20
+ describe 'RSpecFailure' do
21
+ it 'fails' do
22
+ expect(:foo).to eq :bar
23
+ end
22
24
  end
23
25
  end