test-queue 0.2.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.travis.yml +18 -0
- data/Gemfile +2 -0
- data/Gemfile-cucumber1-3 +4 -0
- data/Gemfile-cucumber1-3.lock +33 -0
- data/Gemfile-cucumber2-4 +4 -0
- data/Gemfile-cucumber2-4.lock +37 -0
- data/Gemfile-minitest4.lock +4 -23
- data/Gemfile-minitest5 +3 -0
- data/Gemfile-minitest5.lock +19 -0
- data/Gemfile-rspec2-1 +3 -0
- data/Gemfile-rspec2-1.lock +27 -0
- data/Gemfile-rspec3-0.lock +4 -13
- data/Gemfile-rspec3-1.lock +4 -13
- data/Gemfile-rspec3-2.lock +4 -13
- data/Gemfile-testunit.lock +4 -27
- data/Gemfile.lock +4 -1
- data/README.md +4 -0
- data/bin/minitest-queue +0 -1
- data/bin/testunit-queue +0 -1
- data/lib/test_queue/iterator.rb +41 -10
- data/lib/test_queue/runner.rb +167 -58
- data/lib/test_queue/runner/cucumber.rb +81 -12
- data/lib/test_queue/runner/minitest.rb +0 -4
- data/lib/test_queue/runner/minitest4.rb +25 -2
- data/lib/test_queue/runner/minitest5.rb +36 -11
- data/lib/test_queue/runner/rspec.rb +55 -7
- data/lib/test_queue/runner/rspec2.rb +11 -8
- data/lib/test_queue/runner/rspec3.rb +10 -7
- data/lib/test_queue/runner/sample.rb +0 -2
- data/lib/test_queue/runner/testunit.rb +25 -7
- data/lib/test_queue/stats.rb +95 -0
- data/lib/test_queue/test_framework.rb +29 -0
- data/script/bootstrap +12 -0
- data/script/cibuild +19 -0
- data/script/spec +7 -0
- data/spec/stats_spec.rb +76 -0
- data/test-queue.gemspec +1 -4
- data/test/cucumber.bats +57 -0
- data/test/minitest4.bats +34 -0
- data/test/minitest5.bats +111 -0
- data/test/rspec.bats +38 -0
- data/{features → test/samples/features}/bad.feature +0 -0
- data/{features → test/samples/features}/sample.feature +0 -0
- data/{features → test/samples/features}/sample2.feature +0 -0
- data/{features → test/samples/features}/step_definitions/common.rb +5 -1
- data/test/{sample_minispec.rb → samples/sample_minispec.rb} +6 -0
- data/test/{sample_minitest4.rb → samples/sample_minitest4.rb} +5 -3
- data/test/{sample_minitest5.rb → samples/sample_minitest5.rb} +5 -3
- data/test/{sample_spec.rb → samples/sample_spec.rb} +5 -3
- data/test/samples/sample_split_spec.rb +17 -0
- data/test/{sample_testunit.rb → samples/sample_testunit.rb} +5 -3
- data/test/testlib.bash +81 -0
- data/test/testunit.bats +20 -0
- metadata +40 -60
- data/test-multi.sh +0 -8
- data/test.sh +0 -23
data/script/bootstrap
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
set -ex
|
4
|
+
|
5
|
+
cd "$(dirname "$0")/.."
|
6
|
+
ROOT=$(pwd)
|
7
|
+
|
8
|
+
rm -rf tmp
|
9
|
+
mkdir tmp
|
10
|
+
mkdir -p vendor/bats
|
11
|
+
curl --silent --location --show-error https://github.com/sstephenson/bats/archive/v0.4.0.tar.gz | tar -xz -C tmp
|
12
|
+
tmp/bats-0.4.0/install.sh "$ROOT/vendor/bats"
|
data/script/cibuild
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/bin/sh
|
2
|
+
|
3
|
+
set -ex
|
4
|
+
|
5
|
+
if [ "$SUITE" = "ruby" ]; then
|
6
|
+
exec script/spec
|
7
|
+
fi
|
8
|
+
|
9
|
+
export TEST_QUEUE_WORKERS=2 TEST_QUEUE_VERBOSE=1
|
10
|
+
|
11
|
+
if [ -f "Gemfile-$SUITE" ]; then
|
12
|
+
export BUNDLE_GEMFILE="Gemfile-$SUITE"
|
13
|
+
else
|
14
|
+
export BUNDLE_GEMFILE=Gemfile
|
15
|
+
fi
|
16
|
+
|
17
|
+
bundle install
|
18
|
+
|
19
|
+
vendor/bats/bin/bats test
|
data/script/spec
ADDED
data/spec/stats_spec.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
require "fileutils"
|
2
|
+
require "tempfile"
|
3
|
+
require "test_queue/stats"
|
4
|
+
|
5
|
+
RSpec.describe TestQueue::Stats do
|
6
|
+
before do
|
7
|
+
Tempfile.open("test_queue_stats") do |f|
|
8
|
+
@path = f.path
|
9
|
+
f.close!
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
after do
|
14
|
+
FileUtils.rm_f(@path)
|
15
|
+
end
|
16
|
+
|
17
|
+
describe "#initialize" do
|
18
|
+
it "ignores empty stats files" do
|
19
|
+
File.write(@path, "")
|
20
|
+
stats = TestQueue::Stats.new(@path)
|
21
|
+
expect(stats.all_suites).to be_empty
|
22
|
+
end
|
23
|
+
|
24
|
+
it "ignores invalid data in the stats files" do
|
25
|
+
File.write(@path, "this is not marshal data")
|
26
|
+
stats = TestQueue::Stats.new(@path)
|
27
|
+
expect(stats.all_suites).to be_empty
|
28
|
+
end
|
29
|
+
|
30
|
+
it "ignores badly-typed data in the stats file" do
|
31
|
+
File.write(@path, Marshal.dump(["heyyy"]))
|
32
|
+
stats = TestQueue::Stats.new(@path)
|
33
|
+
expect(stats.all_suites).to be_empty
|
34
|
+
end
|
35
|
+
|
36
|
+
it "ignores stats files with a wrong version number" do
|
37
|
+
File.write(@path, Marshal.dump({ :version => 1e8, :suites => "boom" }))
|
38
|
+
stats = TestQueue::Stats.new(@path)
|
39
|
+
expect(stats.all_suites).to be_empty
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
it "can save and load data" do
|
44
|
+
stats = TestQueue::Stats.new(@path)
|
45
|
+
time = truncated_now
|
46
|
+
suites = [
|
47
|
+
TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
|
48
|
+
TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time + 5),
|
49
|
+
]
|
50
|
+
stats.record_suites(suites)
|
51
|
+
stats.save
|
52
|
+
|
53
|
+
stats = TestQueue::Stats.new(@path)
|
54
|
+
expect(stats.all_suites.sort_by(&:name)).to eq(suites)
|
55
|
+
end
|
56
|
+
|
57
|
+
it "prunes suites not seen in the last 8 days" do
|
58
|
+
stats = TestQueue::Stats.new(@path)
|
59
|
+
time = truncated_now
|
60
|
+
suites = [
|
61
|
+
TestQueue::Stats::Suite.new("Suite1", "foo.rb", 0.3, time),
|
62
|
+
TestQueue::Stats::Suite.new("Suite2", "bar.rb", 0.5, time - (8 * 24 * 60 * 60) - 2),
|
63
|
+
TestQueue::Stats::Suite.new("Suite3", "baz.rb", 0.6, time - (7 * 24 * 60 * 60)),
|
64
|
+
]
|
65
|
+
stats.record_suites(suites)
|
66
|
+
stats.save
|
67
|
+
|
68
|
+
stats = TestQueue::Stats.new(@path)
|
69
|
+
expect(stats.all_suites.map(&:name).sort).to eq(%w[Suite1 Suite3])
|
70
|
+
end
|
71
|
+
|
72
|
+
# Returns Time.now rounded down to the nearest second.
|
73
|
+
def truncated_now
|
74
|
+
Time.at(Time.now.to_i)
|
75
|
+
end
|
76
|
+
end
|
data/test-queue.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
spec = Gem::Specification.new do |s|
|
2
2
|
s.name = 'test-queue'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.3.0'
|
4
4
|
s.summary = 'parallel test runner'
|
5
5
|
s.description = 'minitest/rspec parallel test runner for CI environments'
|
6
6
|
|
@@ -17,8 +17,5 @@ spec = Gem::Specification.new do |s|
|
|
17
17
|
s.executables << 'testunit-queue'
|
18
18
|
s.executables << 'cucumber-queue'
|
19
19
|
|
20
|
-
s.add_development_dependency 'rspec', '>= 2.13', '< 4.0'
|
21
|
-
s.add_development_dependency 'cucumber', '~> 1.3.10'
|
22
|
-
|
23
20
|
s.files = `git ls-files`.split("\n")
|
24
21
|
end
|
data/test/cucumber.bats
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
load "testlib"
|
2
|
+
|
3
|
+
SCRATCH=tmp/cucumber-tests
|
4
|
+
|
5
|
+
setup() {
|
6
|
+
require_gem "cucumber" ">= 1.0"
|
7
|
+
rm -rf $SCRATCH
|
8
|
+
mkdir -p $SCRATCH
|
9
|
+
}
|
10
|
+
|
11
|
+
teardown() {
|
12
|
+
rm -rf $SCRATCH
|
13
|
+
}
|
14
|
+
|
15
|
+
@test "cucumber-queue succeeds when all features pass" {
|
16
|
+
run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
|
17
|
+
assert_status 0
|
18
|
+
assert_output_contains "Starting test-queue master"
|
19
|
+
}
|
20
|
+
|
21
|
+
@test "cucumber-queue fails when a feature fails" {
|
22
|
+
export FAIL=1
|
23
|
+
run bundle exec cucumber-queue test/samples/features --require test/samples/features/step_definitions
|
24
|
+
assert_status 2
|
25
|
+
assert_output_contains "Starting test-queue master"
|
26
|
+
assert_output_contains "cucumber test/samples/features/bad.feature:2 # Scenario: failure"
|
27
|
+
assert_output_contains "cucumber test/samples/features/sample2.feature:26 # Scenario: failure"
|
28
|
+
}
|
29
|
+
|
30
|
+
@test "cucumber-queue fails when given a missing feature" {
|
31
|
+
run bundle exec cucumber-queue test/samples/does_not_exist.feature --require test/samples/features/step_definitions
|
32
|
+
assert_status 1
|
33
|
+
assert_output_contains "Aborting: Discovering suites failed."
|
34
|
+
}
|
35
|
+
|
36
|
+
@test "cucumber-queue fails when given a malformed feature" {
|
37
|
+
[ -f README.md ]
|
38
|
+
run bundle exec cucumber-queue README.md --require test/samples/features/step_definitions
|
39
|
+
|
40
|
+
# Cucumber 1 and 2 fail in different ways.
|
41
|
+
refute_status 0
|
42
|
+
assert_output_matches 'Aborting: Discovering suites failed\.|README\.md: Parser errors:'
|
43
|
+
}
|
44
|
+
|
45
|
+
@test "cucumber-queue handles test file being deleted" {
|
46
|
+
cp test/samples/features/*.feature $SCRATCH
|
47
|
+
|
48
|
+
run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
|
49
|
+
assert_status 0
|
50
|
+
assert_output_matches "Feature: Foobar$"
|
51
|
+
|
52
|
+
rm $SCRATCH/sample.feature
|
53
|
+
|
54
|
+
run bundle exec cucumber-queue $SCRATCH --require test/samples/features/step_definitions
|
55
|
+
assert_status 0
|
56
|
+
refute_output_matches "Feature: Foobar$"
|
57
|
+
}
|
data/test/minitest4.bats
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
load "testlib"
|
2
|
+
|
3
|
+
setup() {
|
4
|
+
require_gem "minitest" ">= 4.0"
|
5
|
+
}
|
6
|
+
|
7
|
+
@test "minitest-queue succeeds when all tests pass" {
|
8
|
+
run bundle exec minitest-queue ./test/samples/*_minitest4.rb
|
9
|
+
assert_status 0
|
10
|
+
assert_output_contains "Starting test-queue master"
|
11
|
+
}
|
12
|
+
|
13
|
+
@test "minitest-queue fails when a test fails" {
|
14
|
+
export FAIL=1
|
15
|
+
run bundle exec minitest-queue ./test/samples/*_minitest4.rb
|
16
|
+
assert_status 1
|
17
|
+
assert_output_contains "Starting test-queue master"
|
18
|
+
assert_output_contains "1) Failure:"
|
19
|
+
assert_output_contains "MiniTestFailure#test_fail"
|
20
|
+
}
|
21
|
+
|
22
|
+
@test "minitest-queue succeeds when all specs pass" {
|
23
|
+
run bundle exec minitest-queue ./test/samples/*_minispec.rb
|
24
|
+
assert_status 0
|
25
|
+
assert_output_contains "Starting test-queue master"
|
26
|
+
}
|
27
|
+
|
28
|
+
@test "minitest-queue fails when a spec fails" {
|
29
|
+
export FAIL=1
|
30
|
+
run bundle exec minitest-queue ./test/samples/*_minispec.rb
|
31
|
+
assert_status 1
|
32
|
+
assert_output_contains "1) Failure:"
|
33
|
+
assert_output_contains "Meme::when asked about blending possibilities#test_0002_fails"
|
34
|
+
}
|
data/test/minitest5.bats
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
load "testlib"
|
2
|
+
|
3
|
+
SCRATCH=tmp/minitest5-tests
|
4
|
+
|
5
|
+
setup() {
|
6
|
+
require_gem "minitest" ">= 5.0"
|
7
|
+
rm -rf $SCRATCH
|
8
|
+
mkdir -p $SCRATCH
|
9
|
+
}
|
10
|
+
|
11
|
+
teardown() {
|
12
|
+
rm -rf $SCRATCH
|
13
|
+
}
|
14
|
+
|
15
|
+
@test "minitest-queue (minitest5) succeeds when all tests pass" {
|
16
|
+
run bundle exec minitest-queue ./test/samples/*_minitest5.rb
|
17
|
+
assert_status 0
|
18
|
+
assert_output_contains "Starting test-queue master"
|
19
|
+
}
|
20
|
+
|
21
|
+
@test "minitest-queue (minitest5) fails when a test fails" {
|
22
|
+
export FAIL=1
|
23
|
+
run bundle exec minitest-queue ./test/samples/*_minitest5.rb
|
24
|
+
assert_status 1
|
25
|
+
assert_output_contains "Starting test-queue master"
|
26
|
+
assert_output_contains "1) Failure:"
|
27
|
+
assert_output_contains "MiniTestFailure#test_fail"
|
28
|
+
}
|
29
|
+
|
30
|
+
@test "TEST_QUEUE_FORCE whitelists certain tests" {
|
31
|
+
export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MiniTestSleep21,MiniTestSleep8"
|
32
|
+
run bundle exec minitest-queue ./test/samples/*_minitest5.rb
|
33
|
+
assert_status 0
|
34
|
+
assert_output_contains "Starting test-queue master"
|
35
|
+
assert_output_contains "MiniTestSleep21"
|
36
|
+
assert_output_contains "MiniTestSleep8"
|
37
|
+
refute_output_contains "MiniTestSleep9"
|
38
|
+
}
|
39
|
+
|
40
|
+
@test "multi-master succeeds when all tests pass" {
|
41
|
+
export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
|
42
|
+
TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
|
43
|
+
sleep 0.1
|
44
|
+
TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
|
45
|
+
wait
|
46
|
+
|
47
|
+
assert_status 0
|
48
|
+
assert_output_contains "Starting test-queue master"
|
49
|
+
}
|
50
|
+
|
51
|
+
@test "multi-master fails when a test fails" {
|
52
|
+
export FAIL=1
|
53
|
+
export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1)
|
54
|
+
TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec minitest-queue ./test/samples/sample_minitest5.rb || true &
|
55
|
+
sleep 0.1
|
56
|
+
TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
|
57
|
+
wait
|
58
|
+
|
59
|
+
assert_status 1
|
60
|
+
assert_output_contains "Starting test-queue master"
|
61
|
+
assert_output_contains "1) Failure:"
|
62
|
+
assert_output_contains "MiniTestFailure#test_fail"
|
63
|
+
}
|
64
|
+
|
65
|
+
@test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" {
|
66
|
+
export TEST_QUEUE_WORKERS=0
|
67
|
+
run bundle exec minitest-queue ./test/samples/sample_minitest5.rb
|
68
|
+
assert_status 1
|
69
|
+
assert_output_contains "Worker count (0) must be greater than 0"
|
70
|
+
}
|
71
|
+
|
72
|
+
@test "minitest-queue fails when given a missing test file" {
|
73
|
+
run bundle exec minitest-queue ./test/samples/does_not_exist.rb
|
74
|
+
assert_status 1
|
75
|
+
assert_output_contains "Aborting: Discovering suites failed"
|
76
|
+
}
|
77
|
+
|
78
|
+
@test "minitest-queue fails when given a malformed test file" {
|
79
|
+
[ -f README.md ]
|
80
|
+
run bundle exec minitest-queue README.md
|
81
|
+
assert_status 1
|
82
|
+
assert_output_contains "Aborting: Discovering suites failed"
|
83
|
+
}
|
84
|
+
|
85
|
+
@test "minitest-queue handles test file being deleted" {
|
86
|
+
cp test/samples/sample_mini{test5,spec}.rb $SCRATCH
|
87
|
+
|
88
|
+
run bundle exec minitest-queue $SCRATCH/*
|
89
|
+
assert_status 0
|
90
|
+
assert_output_contains "Meme::when asked about blending possibilities"
|
91
|
+
|
92
|
+
rm $SCRATCH/sample_minispec.rb
|
93
|
+
|
94
|
+
run bundle exec minitest-queue $SCRATCH/*
|
95
|
+
assert_status 0
|
96
|
+
refute_output_contains "Meme::when asked about blending possibilities"
|
97
|
+
}
|
98
|
+
|
99
|
+
@test "minitest-queue handles suites changing inside a file" {
|
100
|
+
cp test/samples/sample_minispec.rb $SCRATCH
|
101
|
+
|
102
|
+
run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
|
103
|
+
assert_status 0
|
104
|
+
assert_output_contains "Meme::when asked about blending possibilities"
|
105
|
+
|
106
|
+
sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/sample_minispec.rb
|
107
|
+
|
108
|
+
run bundle exec minitest-queue $SCRATCH/sample_minispec.rb
|
109
|
+
assert_status 0
|
110
|
+
assert_output_contains "Meme2::when asked about blending possibilities"
|
111
|
+
}
|
data/test/rspec.bats
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
load "testlib"
|
2
|
+
|
3
|
+
setup() {
|
4
|
+
require_gem "rspec" ">= 2.0"
|
5
|
+
}
|
6
|
+
|
7
|
+
@test "rspec-queue succeeds when all specs pass" {
|
8
|
+
run bundle exec rspec-queue ./test/samples/sample_spec.rb
|
9
|
+
assert_status 0
|
10
|
+
assert_output_contains "Starting test-queue master"
|
11
|
+
}
|
12
|
+
|
13
|
+
@test "rspec-queue fails when a spec fails" {
|
14
|
+
export FAIL=1
|
15
|
+
run bundle exec rspec-queue ./test/samples/sample_spec.rb
|
16
|
+
assert_status 1
|
17
|
+
assert_output_contains "1) RSpecFailure fails"
|
18
|
+
assert_output_contains "Failure/Error: expect(:foo).to eq :bar"
|
19
|
+
}
|
20
|
+
|
21
|
+
@test "TEST_QUEUE_SPLIT_GROUPS splits splittable groups" {
|
22
|
+
export TEST_QUEUE_SPLIT_GROUPS=true
|
23
|
+
run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
|
24
|
+
assert_status 0
|
25
|
+
|
26
|
+
assert_output_matches '\[ 1\] +1 example, 0 failures'
|
27
|
+
assert_output_matches '\[ 2\] +1 example, 0 failures'
|
28
|
+
}
|
29
|
+
|
30
|
+
@test "TEST_QUEUE_SPLIT_GROUPS does not split unsplittable groups" {
|
31
|
+
export TEST_QUEUE_SPLIT_GROUPS=true
|
32
|
+
export NOSPLIT=1
|
33
|
+
run bundle exec rspec-queue ./test/samples/sample_split_spec.rb
|
34
|
+
assert_status 0
|
35
|
+
|
36
|
+
assert_output_contains "2 examples, 0 failures"
|
37
|
+
assert_output_contains "0 examples, 0 failures"
|
38
|
+
}
|
File without changes
|
File without changes
|
File without changes
|