parallel_cucumber 0.1.22 → 0.2.0.pre.36
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +27 -7
- data/bin/parallel_cucumber +1 -1
- data/lib/parallel_cucumber.rb +10 -51
- data/lib/parallel_cucumber/cli.rb +169 -62
- data/lib/parallel_cucumber/helper/command.rb +87 -0
- data/lib/parallel_cucumber/helper/cucumber.rb +104 -0
- data/lib/parallel_cucumber/helper/multi_delegator.rb +24 -0
- data/lib/parallel_cucumber/helper/processes.rb +42 -0
- data/lib/parallel_cucumber/helper/queue.rb +38 -0
- data/lib/parallel_cucumber/helper/unittest/cucumber_test.rb +10 -0
- data/lib/parallel_cucumber/helper/utils.rb +13 -0
- data/lib/parallel_cucumber/logger.rb +15 -0
- data/lib/parallel_cucumber/main.rb +134 -0
- data/lib/parallel_cucumber/status.rb +8 -0
- data/lib/parallel_cucumber/version.rb +2 -2
- data/lib/parallel_cucumber/worker.rb +239 -0
- metadata +50 -14
- data/lib/parallel_cucumber/grouper.rb +0 -95
- data/lib/parallel_cucumber/result_formatter.rb +0 -72
- data/lib/parallel_cucumber/runner.rb +0 -121
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9a409cddb7ad8882664b62866dd1d9ffb39caa9d
|
4
|
+
data.tar.gz: 0224ea5dd38f661176160e0534f1b5c673f9d147
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 155b9bd2377687099c769731f2cf576d4e8984883745adadc08a48ac6403ea0f6e779714c0ade86b2dc816f469733ce78cb22c7826b195c6aed74df7fb3cca33
|
7
|
+
data.tar.gz: bd51ffc599151b873599bff76353b3c565f878d18262b6d1c981b364eb708ac3fa6e3e89e611987124a56d110337873d0a2b498d07a57a59944033a3bcf9c8e5
|
data/README.md
CHANGED
@@ -1,15 +1,35 @@
|
|
1
1
|
# Parallel Cucumber
|
2
2
|
|
3
|
+
### Usage
|
4
|
+
|
3
5
|
```
|
4
6
|
Usage: parallel_cucumber [options] [ [FILE|DIR|URL][:LINE[:LINE]*] ]
|
5
7
|
Example: parallel_cucumber -n 4 -o "-f pretty -f html -o report.html" examples/i18n/en/features
|
6
|
-
-n
|
7
|
-
-o "
|
8
|
-
--cucumber
|
9
|
-
|
10
|
-
-
|
11
|
-
|
12
|
-
|
8
|
+
-n WORKERS How many workers to use. Default is 1 or longest list in -e
|
9
|
+
-o, --cucumber-options "OPTIONS" Run cucumber with these options
|
10
|
+
--test-command COMMAND Command to run for test phase, default cucumber
|
11
|
+
--pre-batch-check COMMAND Command causing worker to quit on exit failure
|
12
|
+
-e, --env-variables JSON Set additional environment variables to processes
|
13
|
+
--batch-size SIZE How many tests each worker takes from queue at once. Default is 1
|
14
|
+
-q ARRAY, `url,name` Url for TCP connection: `redis://[password]@[hostname]:[port]/[db]` (password, port and database are optional), for unix socket connection: `unix://[path to Redis socket]`. Default is redis://127.0.0.1:6379 and name is `queue`
|
15
|
+
--queue-connection-params
|
16
|
+
--setup-worker SCRIPT Execute SCRIPT before each worker
|
17
|
+
--teardown-worker SCRIPT Execute SCRIPT after each worker
|
18
|
+
--worker-delay SECONDS Delay before next worker starting. Could be used for avoiding 'spikes' in CPU and RAM usage Default is 0
|
19
|
+
--batch-timeout SECONDS Timeout for each batch of tests. Default is 600
|
20
|
+
--debug Print more debug information
|
13
21
|
-v, --version Show version
|
14
22
|
-h, --help Show this
|
15
23
|
```
|
24
|
+
|
25
|
+
### Reports
|
26
|
+
|
27
|
+
```yaml
|
28
|
+
# config/cucumber.yaml
|
29
|
+
|
30
|
+
<% test_batch_id = "#{ENV['TEST_BATCH_ID']}" %>
|
31
|
+
|
32
|
+
parallel_reports: >
|
33
|
+
--format html --out reports/cukes_<%= test_batch_id %>.html
|
34
|
+
--format junit --out reports/junit_<%= test_batch_id %>/
|
35
|
+
```
|
data/bin/parallel_cucumber
CHANGED
data/lib/parallel_cucumber.rb
CHANGED
@@ -1,53 +1,12 @@
|
|
1
|
-
require 'parallel'
|
2
|
-
|
3
1
|
require 'parallel_cucumber/cli'
|
4
|
-
require 'parallel_cucumber/
|
5
|
-
require 'parallel_cucumber/
|
6
|
-
require 'parallel_cucumber/
|
2
|
+
require 'parallel_cucumber/helper/command'
|
3
|
+
require 'parallel_cucumber/helper/cucumber'
|
4
|
+
require 'parallel_cucumber/helper/multi_delegator'
|
5
|
+
require 'parallel_cucumber/helper/processes'
|
6
|
+
require 'parallel_cucumber/helper/queue'
|
7
|
+
require 'parallel_cucumber/helper/utils'
|
8
|
+
require 'parallel_cucumber/logger'
|
9
|
+
require 'parallel_cucumber/main'
|
10
|
+
require 'parallel_cucumber/status'
|
7
11
|
require 'parallel_cucumber/version'
|
8
|
-
|
9
|
-
module ParallelCucumber
|
10
|
-
class << self
|
11
|
-
def run_tests_in_parallel(options)
|
12
|
-
number_of_processes = options[:n]
|
13
|
-
test_results = nil
|
14
|
-
|
15
|
-
report_time_taken do
|
16
|
-
groups = Grouper.feature_groups(options, number_of_processes)
|
17
|
-
threads = groups.size
|
18
|
-
completed = []
|
19
|
-
|
20
|
-
on_finish = lambda do |_item, index, _result|
|
21
|
-
completed.push(index)
|
22
|
-
remaining_threads = ((0...threads).to_a - completed).sort
|
23
|
-
puts "Thread #{index} has finished. Remaining(#{remaining_threads.count}): #{remaining_threads.join(', ')}"
|
24
|
-
end
|
25
|
-
|
26
|
-
test_results = Parallel.map_with_index(
|
27
|
-
groups,
|
28
|
-
in_threads: threads,
|
29
|
-
finish: on_finish
|
30
|
-
) do |group, index|
|
31
|
-
Runner.new(options).run_tests(index, group)
|
32
|
-
end
|
33
|
-
puts 'All threads are complete'
|
34
|
-
ResultFormatter.report_results(test_results)
|
35
|
-
end
|
36
|
-
exit(1) if any_test_failed?(test_results)
|
37
|
-
end
|
38
|
-
|
39
|
-
private
|
40
|
-
|
41
|
-
def any_test_failed?(test_results)
|
42
|
-
test_results.any? { |result| result[:exit_status] != 0 }
|
43
|
-
end
|
44
|
-
|
45
|
-
def report_time_taken
|
46
|
-
start = Time.now
|
47
|
-
yield
|
48
|
-
time_in_sec = Time.now - start
|
49
|
-
mm, ss = time_in_sec.divmod(60)
|
50
|
-
puts "\nTook #{mm} Minutes, #{ss.round(2)} Seconds"
|
51
|
-
end
|
52
|
-
end # class
|
53
|
-
end # ParallelCucumber
|
12
|
+
require 'parallel_cucumber/worker'
|
@@ -1,79 +1,186 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'optparse'
|
3
|
+
require 'date'
|
3
4
|
|
4
5
|
module ParallelCucumber
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
6
|
+
class Cli
|
7
|
+
DEFAULTS = {
|
8
|
+
batch_size: 1,
|
9
|
+
batch_timeout: 600,
|
10
|
+
setup_timeout: 30,
|
11
|
+
cucumber_options: '',
|
12
|
+
debug: false,
|
13
|
+
log_dir: '.',
|
14
|
+
log_decoration: {},
|
15
|
+
env_variables: {},
|
16
|
+
n: 0, # Default: computed from longest list in json parameters, minimum 1.
|
17
|
+
queue_connection_params: ['redis://127.0.0.1:6379', DateTime.now.strftime('queue-%Y%m%d%H%M%S')],
|
18
|
+
worker_delay: 0,
|
19
|
+
test_command: 'cucumber'
|
20
|
+
}.freeze
|
19
21
|
|
20
|
-
|
22
|
+
def initialize(argv)
|
23
|
+
@argv = argv
|
24
|
+
@logger = ParallelCucumber::CustomLogger.new(STDOUT)
|
25
|
+
@logger.progname = 'CLI'
|
26
|
+
@logger.level = if @argv.include?('--debug')
|
27
|
+
ParallelCucumber::CustomLogger::DEBUG
|
28
|
+
else
|
29
|
+
ParallelCucumber::CustomLogger::INFO
|
30
|
+
end
|
31
|
+
end
|
21
32
|
|
22
|
-
|
23
|
-
|
33
|
+
def run
|
34
|
+
options = parse_options!(@argv)
|
35
|
+
message = <<-LOG
|
36
|
+
Running parallel_cucumber with options: #{options.map { |k, v| "#{k}=#{v}" }.join(', ')}
|
37
|
+
LOG
|
38
|
+
@logger.debug(message)
|
39
|
+
ParallelCucumber::Main.new(options).run
|
40
|
+
end
|
24
41
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
42
|
-
opts.on('-s', '--setup-script [SCRIPT]', 'Execute SCRIPT before each process') do |script|
|
43
|
-
check_script(script)
|
44
|
-
options[:setup_script] = File.expand_path(script)
|
42
|
+
private
|
43
|
+
|
44
|
+
def parse_options!(argv)
|
45
|
+
options = DEFAULTS.dup
|
46
|
+
|
47
|
+
option_parser = OptionParser.new do |opts|
|
48
|
+
opts.banner = [
|
49
|
+
'Usage: parallel_cucumber [options] [ [FILE|DIR|URL][:LINE[:LINE]*] ]',
|
50
|
+
'Example: parallel_cucumber -n 4 -o "-f pretty -f html -o report.html" examples/i18n/en/features'
|
51
|
+
].join("\n")
|
52
|
+
|
53
|
+
opts.on('-n WORKERS', Integer, 'How many workers to use. Default is 1 or longest list in -e') do |n|
|
54
|
+
if n < 1
|
55
|
+
puts "The minimum number of processes is 1 but given: '#{n}'"
|
56
|
+
exit 1
|
45
57
|
end
|
46
|
-
|
47
|
-
|
48
|
-
|
58
|
+
options[:n] = n
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.on('-o', '--cucumber-options "OPTIONS"', 'Run cucumber with these options') do |cucumber_options|
|
62
|
+
options[:cucumber_options] = cucumber_options
|
63
|
+
end
|
64
|
+
|
65
|
+
opts.on('--test-command COMMAND',
|
66
|
+
"Command to run for test phase, default #{DEFAULTS[:test_command]}") do |test_command|
|
67
|
+
options[:test_command] = test_command
|
68
|
+
end
|
69
|
+
|
70
|
+
opts.on('--pre-batch-check COMMAND', 'Command causing worker to quit on exit failure') do |pre_check|
|
71
|
+
options[:pre_check] = pre_check
|
72
|
+
end
|
73
|
+
|
74
|
+
options[:pretty] = '--format pretty'
|
75
|
+
opts.on('--no-pretty', "Suppress the default 'pretty' formatter directed at stdout") do
|
76
|
+
options[:pretty] = ''
|
77
|
+
end
|
78
|
+
|
79
|
+
opts.on('--log-dir DIR', 'Directory for worker logfiles') do |log_dir|
|
80
|
+
options[:log_dir] = log_dir
|
81
|
+
end
|
82
|
+
|
83
|
+
opts.on('--log-decoration JSON', 'Block quoting for logs, e.g. {start: "#start %s", end: "#end %s"}') do |json|
|
84
|
+
options[:log_decoration] = begin
|
85
|
+
JSON.parse(json)
|
86
|
+
rescue JSON::ParserError
|
87
|
+
puts 'Log block quoting not in JSON format. Did you forget to escape the quotes?'
|
88
|
+
raise
|
49
89
|
end
|
50
|
-
|
51
|
-
|
90
|
+
end
|
91
|
+
|
92
|
+
opts.on('--summary JSON', 'Summary files, e.g. {failed: "./failed.txt", unknown: "./unknown.txt"}') do |json|
|
93
|
+
options[:summary] = begin
|
94
|
+
JSON.parse(json)
|
95
|
+
rescue JSON::ParserError
|
96
|
+
puts 'Log block quoting not in JSON format. Did you forget to escape the quotes?'
|
97
|
+
raise
|
52
98
|
end
|
53
|
-
|
54
|
-
|
55
|
-
|
99
|
+
end
|
100
|
+
|
101
|
+
opts.on('-e', '--env-variables JSON', 'Set additional environment variables to processes') do |env_vars|
|
102
|
+
options[:env_variables] = begin
|
103
|
+
JSON.parse(env_vars)
|
104
|
+
rescue JSON::ParserError
|
105
|
+
puts 'Additional environment variables not in JSON format. Did you forget to escape the quotes?'
|
106
|
+
raise
|
56
107
|
end
|
57
|
-
|
58
|
-
|
59
|
-
|
108
|
+
end
|
109
|
+
|
110
|
+
help_message = "How many tests each worker takes from queue at once. Default is #{DEFAULTS[:batch_size]}"
|
111
|
+
opts.on('--batch-size SIZE', Integer, help_message.gsub(/\s+/, ' ').strip) do |batch_size|
|
112
|
+
if batch_size < 1
|
113
|
+
puts "The minimum batch size is 1 but given: '#{batch_size}'"
|
114
|
+
exit 1
|
60
115
|
end
|
116
|
+
options[:batch_size] = batch_size
|
61
117
|
end
|
62
118
|
|
63
|
-
|
64
|
-
|
119
|
+
help_message = <<-TEXT
|
120
|
+
`url,name`
|
121
|
+
Url for TCP connection:
|
122
|
+
`redis://[password]@[hostname]:[port]/[db]` (password, port and database are optional),
|
123
|
+
for unix socket connection: `unix://[path to Redis socket]`.
|
124
|
+
Default is redis://127.0.0.1:6379 and name is `queue`
|
125
|
+
TEXT
|
126
|
+
opts.on('-q', '--queue-connection-params ARRAY', Array, help_message.gsub(/\s+/, ' ').strip) do |params|
|
127
|
+
options[:queue_connection_params] = params
|
128
|
+
end
|
65
129
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
puts option_parser.help
|
70
|
-
exit 1
|
71
|
-
end
|
130
|
+
opts.on('--setup-worker SCRIPT', 'Execute SCRIPT before each worker') do |script|
|
131
|
+
options[:setup_worker] = script
|
132
|
+
end
|
72
133
|
|
73
|
-
|
74
|
-
|
75
|
-
|
134
|
+
opts.on('--teardown-worker SCRIPT', 'Execute SCRIPT after each worker') do |script|
|
135
|
+
options[:teardown_worker] = script
|
136
|
+
end
|
137
|
+
|
138
|
+
help_message = <<-TEXT
|
139
|
+
Delay before next worker starting.
|
140
|
+
Could be used for avoiding 'spikes' in CPU and RAM usage
|
141
|
+
Default is #{DEFAULTS[:worker_delay]}
|
142
|
+
TEXT
|
143
|
+
opts.on('--worker-delay SECONDS', Float, help_message.gsub(/\s+/, ' ').strip) do |worker_delay|
|
144
|
+
options[:worker_delay] = worker_delay
|
145
|
+
end
|
146
|
+
|
147
|
+
help_message = <<-TEXT
|
148
|
+
Timeout for each batch of tests. Default is #{DEFAULTS[:batch_timeout]}
|
149
|
+
TEXT
|
150
|
+
opts.on('--batch-timeout SECONDS', Float, help_message.gsub(/\s+/, ' ').strip) do |batch_timeout|
|
151
|
+
options[:batch_timeout] = batch_timeout
|
152
|
+
end
|
153
|
+
|
154
|
+
help_message = <<-TEXT
|
155
|
+
Timeout for each worker's set-up phase. Default is #{DEFAULTS[:setup_timeout]}
|
156
|
+
TEXT
|
157
|
+
opts.on('--setup-timeout SECONDS', Float, help_message.gsub(/\s+/, ' ').strip) do |setup_timeout|
|
158
|
+
options[:setup_timeout] = setup_timeout
|
159
|
+
end
|
160
|
+
|
161
|
+
opts.on('--debug', 'Print more debug information') do |debug|
|
162
|
+
options[:debug] = debug
|
163
|
+
end
|
164
|
+
|
165
|
+
opts.on('-v', '--version', 'Show version') do
|
166
|
+
puts ParallelCucumber::VERSION
|
167
|
+
exit 0
|
168
|
+
end
|
169
|
+
|
170
|
+
opts.on('-h', '--help', 'Show this') do
|
171
|
+
puts opts
|
172
|
+
exit 0
|
173
|
+
end
|
76
174
|
end
|
77
|
-
|
78
|
-
|
79
|
-
|
175
|
+
|
176
|
+
option_parser.parse!(argv)
|
177
|
+
options[:cucumber_args] = argv
|
178
|
+
|
179
|
+
options
|
180
|
+
rescue OptionParser::InvalidOption => e
|
181
|
+
puts "Unknown option #{e}"
|
182
|
+
puts option_parser.help
|
183
|
+
exit 1
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module ParallelCucumber
|
2
|
+
module Helper
|
3
|
+
module Command
|
4
|
+
class << self
|
5
|
+
def wrap_block(log_decoration, block_name, logger)
|
6
|
+
logger << format(log_decoration['start'] + "\n", block_name) if log_decoration['start']
|
7
|
+
yield
|
8
|
+
ensure
|
9
|
+
logger << format(log_decoration['end'] + "\n", block_name) if log_decoration['end']
|
10
|
+
end
|
11
|
+
|
12
|
+
def exec_command(env, desc, script, log_file, logger, log_decoration = {}, timeout = 30) # rubocop:disable Metrics/ParameterLists, Metrics/LineLength
|
13
|
+
file = File.open(log_file)
|
14
|
+
file.seek(0, File::SEEK_END)
|
15
|
+
full_script = "#{script} >>#{log_file} 2>&1"
|
16
|
+
env_string = env.map { |k, v| "#{k}=#{v}" }.sort.join(' ')
|
17
|
+
message = <<-LOG
|
18
|
+
Running command `#{full_script}` with environment variables: #{env_string}
|
19
|
+
LOG
|
20
|
+
logger.debug(message)
|
21
|
+
pstat = nil
|
22
|
+
pout = nil
|
23
|
+
begin
|
24
|
+
completed = Timeout.timeout(timeout) do
|
25
|
+
pin, pout, pstat = Open3.popen2e(env, full_script)
|
26
|
+
logger.debug("Command has pid #{pstat[:pid]}")
|
27
|
+
pin.close
|
28
|
+
out = pout.readlines.join("\n") # Not expecting anything in 'out' due to redirection, but...
|
29
|
+
pout.close
|
30
|
+
pstat.value # N.B. Await process termination
|
31
|
+
"Command completed #{pstat.value} and expecting '#{out}' to be empty due to redirects"
|
32
|
+
end
|
33
|
+
logger.debug(completed)
|
34
|
+
# system("lsof #{log_file} >> #{log_file} 2>&1")
|
35
|
+
system("ps -axf | grep '#{pstat[:pid]}\\s' >> #{log_file}")
|
36
|
+
return pstat.value.success?
|
37
|
+
rescue Timeout::Error
|
38
|
+
pout.close
|
39
|
+
logger.debug("Timeout, so trying SIGINT #{pstat[:pid]}")
|
40
|
+
wait_sigint = 15
|
41
|
+
logger.error("Timeout #{timeout}s was reached. Sending SIGINT(2), then waiting up to #{wait_sigint}s")
|
42
|
+
tree = Helper::Processes.ps_tree
|
43
|
+
begin
|
44
|
+
Helper::Processes.kill_tree('SIGINT', pstat[:pid].to_s, tree)
|
45
|
+
timed_out = wait_sigint.times do |t|
|
46
|
+
break if Helper::Processes.all_pids_dead?(pstat[:pid].to_s, nil, tree)
|
47
|
+
logger.info("Wait dead #{t}")
|
48
|
+
sleep 1
|
49
|
+
end
|
50
|
+
if timed_out
|
51
|
+
logger.error("Process #{pstat[:pid]} lasted #{wait_sigint}s after SIGINT(2), so SIGKILL(9)! Fatality!")
|
52
|
+
Helper::Processes.kill_tree('SIGKILL', pstat[:pid].to_s, nil, tree)
|
53
|
+
end
|
54
|
+
logger.debug("About to reap root #{pstat[:pid]}")
|
55
|
+
pstat.value # reap root - everything else should be reaped by init.
|
56
|
+
logger.debug("Reaped root #{pstat[:pid]}")
|
57
|
+
# system("lsof #{log_file} >> #{log_file}")
|
58
|
+
logger.debug("Tried SIGKILL #{pstat[:pid]} - hopefully no processes still have #{log_file}!")
|
59
|
+
end
|
60
|
+
rescue => e
|
61
|
+
logger.debug("Exception #{pstat ? pstat[:pid] : "pstat=#{pstat}=nil"}")
|
62
|
+
trace = e.backtrace.join("\n\t").sub("\n\t", ": #{$ERROR_INFO}#{e.class ? " (#{e.class})" : ''}\n\t")
|
63
|
+
logger.error("Threw: for #{full_script}, caused #{trace}")
|
64
|
+
ensure
|
65
|
+
# It's only logging - don't really care if we lose some, though it would be nice if we didn't.
|
66
|
+
if log_decoration['worker_block']
|
67
|
+
prefix = "#{env['TEST_USER']}-w#{env['WORKER_INDEX']}>"
|
68
|
+
block_name = ''
|
69
|
+
if log_decoration['start'] || log_decoration['end']
|
70
|
+
block_name = "#{prefix} #{desc}"
|
71
|
+
prefix = ''
|
72
|
+
end
|
73
|
+
message = ''
|
74
|
+
message << format(log_decoration['start'] + "\n", block_name) if log_decoration['start']
|
75
|
+
message << "#{prefix}#{file.readline}" until file.eof
|
76
|
+
message << format(log_decoration['end'] + "\n", block_name) if log_decoration['end']
|
77
|
+
logger << message
|
78
|
+
file.close
|
79
|
+
end
|
80
|
+
end
|
81
|
+
logger.debug("Unusual termination for command: #{script}")
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|