parallel_cucumber 0.2.16 → 0.2.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/parallel_cucumber.rb +2 -0
- data/lib/parallel_cucumber/cli.rb +6 -13
- data/lib/parallel_cucumber/dsl.rb +4 -0
- data/lib/parallel_cucumber/helper/cucumber/cucumber.rb +27 -20
- data/lib/parallel_cucumber/helper/cucumber/json_status_formatter.rb +7 -1
- data/lib/parallel_cucumber/hooks.rb +17 -5
- data/lib/parallel_cucumber/job.rb +13 -0
- data/lib/parallel_cucumber/main.rb +8 -39
- data/lib/parallel_cucumber/version.rb +1 -1
- data/lib/parallel_cucumber/worker.rb +40 -62
- data/lib/parallel_cucumber/worker_manager.rb +157 -0
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b2a5fcb6c187e21abd99851009a31c5015221e31118e5e16ccbb022467f64614
|
4
|
+
data.tar.gz: f45853fb7908faa80978b52c7e8208b2cec406a4dad87c80949623e615870dcc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21f7385b3c24476db0de9b9890d0e47166cd2ab3d4d5a5c52e8a73e39bc97e58502c50c0faaf4c82f0e250bda4ebe86a401a6b40c5f84b71959f058a9c6bfecb
|
7
|
+
data.tar.gz: eaeb5edcda79f778eb6b3bf1c0af1bfdabe69137422724a79523e9f0288c5e3e8177cfb30b920c227173ffcdf4644272772fb112ac46a214a3d3423a2709d0e9
|
data/lib/parallel_cucumber.rb
CHANGED
@@ -7,8 +7,10 @@ require 'parallel_cucumber/helper/processes'
|
|
7
7
|
require 'parallel_cucumber/helper/queue'
|
8
8
|
require 'parallel_cucumber/helper/utils'
|
9
9
|
require 'parallel_cucumber/hooks'
|
10
|
+
require 'parallel_cucumber/job'
|
10
11
|
require 'parallel_cucumber/logger'
|
11
12
|
require 'parallel_cucumber/main'
|
12
13
|
require 'parallel_cucumber/status'
|
13
14
|
require 'parallel_cucumber/version'
|
15
|
+
require 'parallel_cucumber/worker_manager'
|
14
16
|
require 'parallel_cucumber/worker'
|
@@ -8,7 +8,6 @@ module ParallelCucumber
|
|
8
8
|
batch_size: 1,
|
9
9
|
batch_timeout: 600,
|
10
10
|
setup_timeout: 30,
|
11
|
-
precheck_timeout: 30,
|
12
11
|
batch_error_timeout: 30,
|
13
12
|
cucumber_options: '',
|
14
13
|
debug: false,
|
@@ -19,7 +18,8 @@ module ParallelCucumber
|
|
19
18
|
n: 0, # Default: computed from longest list in json parameters, minimum 1.
|
20
19
|
queue_connection_params: ['redis://127.0.0.1:6379', DateTime.now.strftime('queue-%Y%m%d%H%M%S')],
|
21
20
|
worker_delay: 0,
|
22
|
-
test_command: 'cucumber'
|
21
|
+
test_command: 'cucumber',
|
22
|
+
backup_worker_count: 0
|
23
23
|
}.freeze
|
24
24
|
|
25
25
|
def initialize(argv)
|
@@ -61,6 +61,10 @@ module ParallelCucumber
|
|
61
61
|
options[:n] = n
|
62
62
|
end
|
63
63
|
|
64
|
+
opts.on('--backup-worker-count BACKUP_WORKERS', Integer, 'How many free workers to hold before all tasks are done. Default is none') do |n|
|
65
|
+
options[:backup_worker_count] = n
|
66
|
+
end
|
67
|
+
|
64
68
|
opts.on('-o', '--cucumber-options "OPTIONS"', 'Run cucumber with these options') do |cucumber_options|
|
65
69
|
options[:cucumber_options] = cucumber_options
|
66
70
|
end
|
@@ -85,10 +89,6 @@ module ParallelCucumber
|
|
85
89
|
options[:test_command] = test_command
|
86
90
|
end
|
87
91
|
|
88
|
-
opts.on('--pre-batch-check COMMAND', 'Command causing worker to quit on exit failure') do |pre_check|
|
89
|
-
options[:pre_check] = pre_check
|
90
|
-
end
|
91
|
-
|
92
92
|
opts.on('--log-dir DIR', 'Directory for worker logfiles') do |log_dir|
|
93
93
|
options[:log_dir] = log_dir
|
94
94
|
end
|
@@ -168,13 +168,6 @@ module ParallelCucumber
|
|
168
168
|
options[:batch_timeout] = batch_timeout
|
169
169
|
end
|
170
170
|
|
171
|
-
help_message = <<-TEXT.gsub(/\s+/, ' ').strip
|
172
|
-
Timeout for each test precheck. Default is #{DEFAULTS[:batch_timeout]}
|
173
|
-
TEXT
|
174
|
-
opts.on('--precheck-timeout SECONDS', Float, help_message) do |timeout|
|
175
|
-
options[:precheck_timeout] = timeout
|
176
|
-
end
|
177
|
-
|
178
171
|
help_message = <<-TEXT.gsub(/\s+/, ' ').strip
|
179
172
|
Timeout for each batch_error script. Default is #{DEFAULTS[:batch_error_timeout]}
|
180
173
|
TEXT
|
@@ -26,25 +26,32 @@ module ParallelCucumber
|
|
26
26
|
|
27
27
|
def parse_json_report(json_report)
|
28
28
|
report = JSON.parse(json_report, symbolize_names: true)
|
29
|
-
report.
|
30
|
-
status = case
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
29
|
+
report.each do |scenario, details|
|
30
|
+
report[scenario][:status] = case details[:status]
|
31
|
+
when 'failed'
|
32
|
+
Status::FAILED
|
33
|
+
when 'passed'
|
34
|
+
Status::PASSED
|
35
|
+
when 'pending'
|
36
|
+
Status::PENDING
|
37
|
+
when 'skipped'
|
38
|
+
Status::SKIPPED
|
39
|
+
when 'undefined'
|
40
|
+
Status::UNDEFINED
|
41
|
+
when 'unknown'
|
42
|
+
Status::UNKNOWN
|
43
|
+
else
|
44
|
+
Status::UNKNOWN
|
45
|
+
end
|
46
|
+
end
|
47
|
+
report
|
48
|
+
end
|
49
|
+
|
50
|
+
def unknown_result(tests)
|
51
|
+
res = tests.map do |test|
|
52
|
+
[test.to_sym, {status: ::ParallelCucumber::Status::UNKNOWN}]
|
53
|
+
end
|
54
|
+
res.to_h
|
48
55
|
end
|
49
56
|
|
50
57
|
private
|
@@ -57,7 +64,7 @@ module ParallelCucumber
|
|
57
64
|
options = remove_strict_flag(options)
|
58
65
|
content = nil
|
59
66
|
|
60
|
-
Tempfile.open(%w
|
67
|
+
Tempfile.open(%w[dry-run .json]) do |f|
|
61
68
|
dry_run_options = "--dry-run --format ParallelCucumber::Helper::Cucumber::JsonStatusFormatter --out #{f.path}"
|
62
69
|
|
63
70
|
cmd = "cucumber #{options} #{dry_run_options} #{args_string}"
|
@@ -15,7 +15,13 @@ module ParallelCucumber
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def on_after_test_case(event)
|
18
|
-
|
18
|
+
details = {status: event.result.to_sym}
|
19
|
+
if event.result.respond_to?(:exception)
|
20
|
+
details[:exception_classname] = event.result.exception.class.to_s
|
21
|
+
details[:exception_message] = event.result.exception.message
|
22
|
+
end
|
23
|
+
details[:finish_time] = Time.now.to_i
|
24
|
+
@result[event.test_case.location.to_s] = details
|
19
25
|
end
|
20
26
|
|
21
27
|
def on_finished_testing(*)
|
@@ -1,12 +1,18 @@
|
|
1
1
|
module ParallelCucumber
|
2
2
|
class Hooks
|
3
|
-
@
|
4
|
-
@
|
5
|
-
@
|
6
|
-
@
|
7
|
-
@
|
3
|
+
@worker_health_check ||= []
|
4
|
+
@before_batch_hooks ||= []
|
5
|
+
@after_batch_hooks ||= []
|
6
|
+
@before_workers ||= []
|
7
|
+
@after_workers ||= []
|
8
|
+
@on_batch_error ||= []
|
8
9
|
|
9
10
|
class << self
|
11
|
+
def register_worker_health_check(proc)
|
12
|
+
raise(ArgumentError, 'Please provide a valid callback') unless proc.respond_to?(:call)
|
13
|
+
@worker_health_check << proc
|
14
|
+
end
|
15
|
+
|
10
16
|
def register_before_batch(proc)
|
11
17
|
raise(ArgumentError, 'Please provide a valid callback') unless proc.respond_to?(:call)
|
12
18
|
@before_batch_hooks << proc
|
@@ -32,6 +38,12 @@ module ParallelCucumber
|
|
32
38
|
@on_batch_error << proc
|
33
39
|
end
|
34
40
|
|
41
|
+
def fire_worker_health_check(*args)
|
42
|
+
@worker_health_check.each do |hook|
|
43
|
+
hook.call(*args)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
35
47
|
def fire_before_batch_hooks(*args)
|
36
48
|
@before_batch_hooks.each do |hook|
|
37
49
|
hook.call(*args)
|
@@ -98,7 +98,7 @@ module ParallelCucumber
|
|
98
98
|
|
99
99
|
status_totals = Status.constants.map do |status|
|
100
100
|
status = Status.const_get(status)
|
101
|
-
tests_with_status = results.select { |_t, s| s == status }.keys
|
101
|
+
tests_with_status = results.select { |_t, s| s[:status] == status }.keys
|
102
102
|
[status, tests_with_status]
|
103
103
|
end.to_h
|
104
104
|
|
@@ -149,17 +149,11 @@ module ParallelCucumber
|
|
149
149
|
Helper::Command.wrap_block(@options[:log_decoration],
|
150
150
|
@options[:log_decoration]['worker_block'] || 'workers',
|
151
151
|
@logger) do
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
) do |index|
|
158
|
-
ParallelCucumber::Worker
|
159
|
-
.new(@options, index, @logger)
|
160
|
-
.start(env_for_worker(@options[:env_variables], index))
|
161
|
-
end
|
162
|
-
map.inject(:merge) # Returns hash of file:line to statuses + :worker-index to summary.
|
152
|
+
|
153
|
+
worker_manager = ParallelCucumber::WorkerManager.new(@options, @logger)
|
154
|
+
worker_manager.start(number_of_workers)
|
155
|
+
ensure
|
156
|
+
worker_manager.kill
|
163
157
|
end
|
164
158
|
end
|
165
159
|
|
@@ -169,11 +163,11 @@ module ParallelCucumber
|
|
169
163
|
@logger.info("Inferred worker count #{@options[:n]} from env_variables option")
|
170
164
|
end
|
171
165
|
|
172
|
-
number_of_workers = [@options[:n], count].min
|
166
|
+
number_of_workers = [@options[:n], [@options[:backup_worker_count], count].max].min
|
173
167
|
unless number_of_workers == @options[:n]
|
174
168
|
@logger.info(<<-LOG)
|
175
169
|
Number of workers was overridden to #{number_of_workers}.
|
176
|
-
More workers (#{@options[:n]}) requested than tests (#{count})".
|
170
|
+
More workers (#{@options[:n]}) requested than tests (#{count}). BackupWorkerCount: #{@options[:backup_worker_count]}".
|
177
171
|
LOG
|
178
172
|
end
|
179
173
|
|
@@ -192,30 +186,5 @@ module ParallelCucumber
|
|
192
186
|
end
|
193
187
|
number_of_workers
|
194
188
|
end
|
195
|
-
|
196
|
-
private
|
197
|
-
|
198
|
-
def env_for_worker(env_variables, worker_number)
|
199
|
-
env = env_variables.map do |k, v|
|
200
|
-
case v
|
201
|
-
when String, Numeric, TrueClass, FalseClass
|
202
|
-
[k, v]
|
203
|
-
when Array
|
204
|
-
[k, v[worker_number]]
|
205
|
-
when Hash
|
206
|
-
value = v[worker_number.to_s]
|
207
|
-
[k, value] unless value.nil?
|
208
|
-
when NilClass
|
209
|
-
else
|
210
|
-
raise("Don't know how to set '#{v}'<#{v.class}> to the environment variable '#{k}'")
|
211
|
-
end
|
212
|
-
end.compact.to_h
|
213
|
-
|
214
|
-
# Defaults, if absent in env. Shame 'merge' isn't something non-commutative like 'adopts/defaults'.
|
215
|
-
env = { TEST: 1, TEST_PROCESS_NUMBER: worker_number, WORKER_INDEX: worker_number }.merge(env)
|
216
|
-
|
217
|
-
# Overwrite this if it exists in env.
|
218
|
-
env.merge(PARALLEL_CUCUMBER_EXPORTS: env.keys.join(',')).map { |k, v| [k.to_s, v.to_s] }.to_h
|
219
|
-
end
|
220
189
|
end
|
221
190
|
end
|
@@ -3,38 +3,18 @@ require 'timeout'
|
|
3
3
|
require 'tmpdir' # I loathe Ruby.
|
4
4
|
|
5
5
|
module ParallelCucumber
|
6
|
-
class Tracker
|
7
|
-
def initialize(queue)
|
8
|
-
@queue = queue
|
9
|
-
end
|
10
|
-
|
11
|
-
def status
|
12
|
-
queue_length = @queue.length
|
13
|
-
now = Time.now
|
14
|
-
@full ||= queue_length
|
15
|
-
@start ||= now
|
16
|
-
completed = @full - queue_length
|
17
|
-
elapsed = now - @start
|
18
|
-
estimate = (completed == 0) ? '' : " #{(elapsed * @full / completed).to_i}s est"
|
19
|
-
"#{queue_length}/#{@full} left #{elapsed.to_i}s worker#{estimate}"
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
6
|
class Worker
|
24
7
|
include ParallelCucumber::Helper::Utils
|
25
8
|
|
26
|
-
def initialize(options
|
27
|
-
@batch_size = options[:batch_size]
|
9
|
+
def initialize(options:, index:, stdout_logger:, manager:)
|
28
10
|
@group_by = options[:group_by]
|
29
11
|
@batch_timeout = options[:batch_timeout]
|
30
12
|
@batch_error_timeout = options[:batch_error_timeout]
|
31
|
-
@precheck_timeout = options[:precheck_timeout]
|
32
13
|
@setup_timeout = options[:setup_timeout]
|
33
14
|
@cucumber_options = options[:cucumber_options]
|
34
15
|
@test_command = options[:test_command]
|
35
|
-
@pre_check = options[:pre_check]
|
36
16
|
@index = index
|
37
|
-
@
|
17
|
+
@name = "W#{@index}"
|
38
18
|
@setup_worker = options[:setup_worker]
|
39
19
|
@teardown_worker = options[:teardown_worker]
|
40
20
|
@worker_delay = options[:worker_delay]
|
@@ -43,6 +23,19 @@ module ParallelCucumber
|
|
43
23
|
@log_dir = options[:log_dir]
|
44
24
|
@log_file = "#{@log_dir}/worker_#{index}.log"
|
45
25
|
@stdout_logger = stdout_logger # .sync writes only.
|
26
|
+
@is_busy_running_test = false
|
27
|
+
@jobs_queue = Queue.new
|
28
|
+
@manager = manager
|
29
|
+
end
|
30
|
+
|
31
|
+
attr_reader :index
|
32
|
+
|
33
|
+
def assign_job(instruction)
|
34
|
+
@jobs_queue.enq(instruction)
|
35
|
+
end
|
36
|
+
|
37
|
+
def busy_running_test?
|
38
|
+
@is_busy_running_test && @current_thread.alive?
|
46
39
|
end
|
47
40
|
|
48
41
|
def autoshutting_file
|
@@ -68,6 +61,9 @@ module ParallelCucumber
|
|
68
61
|
end
|
69
62
|
|
70
63
|
def start(env)
|
64
|
+
@current_thread = Thread.current
|
65
|
+
@manager.inform_idle(@name)
|
66
|
+
|
71
67
|
env = env.dup.merge!('WORKER_LOG' => @log_file)
|
72
68
|
|
73
69
|
File.delete(@log_file) if File.exist?(@log_file)
|
@@ -96,33 +92,26 @@ module ParallelCucumber
|
|
96
92
|
begin
|
97
93
|
setup(env)
|
98
94
|
|
99
|
-
queue = ParallelCucumber::Helper::Queue.new(@queue_connection_params)
|
100
|
-
directed_queue = ParallelCucumber::Helper::Queue.new(@queue_connection_params, "_#{@index}")
|
101
|
-
queue_tracker = Tracker.new(queue)
|
102
|
-
|
103
95
|
loop_mm, loop_ss = time_it do
|
104
96
|
loop do
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
97
|
+
job = @jobs_queue.pop(false)
|
98
|
+
case job.type
|
99
|
+
when Job::PRECHECK
|
100
|
+
Hooks.fire_worker_health_check(env)
|
101
|
+
@manager.inform_healthy(@name)
|
102
|
+
when Job::RUN_TESTS
|
103
|
+
run_batch(env, results, running_total, job.details)
|
104
|
+
@manager.inform_idle(@name)
|
105
|
+
when Job::DIE
|
106
|
+
break
|
107
|
+
else
|
108
|
+
raise("Invalid job #{job.inspect}")
|
115
109
|
end
|
116
|
-
batch.compact!
|
117
|
-
batch.sort! # Workaround for https://github.com/cucumber/cucumber-ruby/issues/952
|
118
|
-
break if batch.empty?
|
119
|
-
|
120
|
-
run_batch(env, queue_tracker, results, running_total, batch)
|
121
110
|
end
|
122
111
|
end
|
123
112
|
@logger.debug("Loop took #{loop_mm} minutes #{loop_ss} seconds")
|
124
113
|
@logger.update_into(@stdout_logger)
|
125
|
-
rescue => e
|
114
|
+
rescue StandardError => e
|
126
115
|
trace = e.backtrace.join("\n\t").sub("\n\t", ": #{$ERROR_INFO}#{e.class ? " (#{e.class})" : ''}\n\t")
|
127
116
|
@logger.error("Threw: #{e.inspect} #{trace}")
|
128
117
|
ensure
|
@@ -135,10 +124,10 @@ module ParallelCucumber
|
|
135
124
|
results
|
136
125
|
end
|
137
126
|
|
138
|
-
def run_batch(env,
|
127
|
+
def run_batch(env, results, running_total, tests)
|
128
|
+
@is_busy_running_test = true
|
139
129
|
batch_id = "#{Time.now.to_i}-#{@index}"
|
140
130
|
@logger.debug("Batch ID is #{batch_id}")
|
141
|
-
@logger.info("Took #{tests.count} from the queue (#{queue_tracker.status}): #{tests.join(' ')}")
|
142
131
|
|
143
132
|
batch_mm, batch_ss = time_it do
|
144
133
|
begin
|
@@ -158,28 +147,17 @@ module ParallelCucumber
|
|
158
147
|
process_results(batch_results, tests)
|
159
148
|
running_totals(batch_results, running_total)
|
160
149
|
results.merge!(batch_results)
|
150
|
+
@is_busy_running_test = false
|
161
151
|
end
|
162
152
|
ensure
|
163
153
|
@logger.debug("Batch #{batch_id} took #{batch_mm} minutes #{batch_ss} seconds")
|
164
154
|
@logger.update_into(@stdout_logger)
|
165
155
|
end
|
166
156
|
|
167
|
-
def precheck(env)
|
168
|
-
return 'default no-op pre_check' unless @pre_check
|
169
|
-
begin
|
170
|
-
return Helper::Command.exec_command(
|
171
|
-
env, 'precheck', @pre_check, @logger, @log_decoration, timeout: @precheck_timeout, capture: true
|
172
|
-
)
|
173
|
-
rescue
|
174
|
-
@logger.error('Pre-check failed: quitting immediately')
|
175
|
-
raise 'Pre-check failed: quitting immediately'
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
157
|
def running_totals(batch_results, running_total)
|
180
158
|
batch_info = Status.constants.map do |status|
|
181
159
|
status = Status.const_get(status)
|
182
|
-
[status, batch_results.select { |_t, s| s == status }.keys]
|
160
|
+
[status, batch_results.select { |_t, s| s[:status] == status }.keys]
|
183
161
|
end.to_h
|
184
162
|
batch_info.each do |s, tt|
|
185
163
|
@logger.info("#{s.to_s.upcase} #{tt.count} tests: #{tt.join(' ')}") unless tt.empty?
|
@@ -194,7 +172,7 @@ module ParallelCucumber
|
|
194
172
|
test_syms = tests.map(&:to_sym)
|
195
173
|
unrun = test_syms - batch_keys
|
196
174
|
surfeit = batch_keys - test_syms
|
197
|
-
unrun.each { |test| batch_results[test] = Status::UNKNOWN }
|
175
|
+
unrun.each { |test| batch_results[test][:status] = Status::UNKNOWN }
|
198
176
|
surfeit.each { |test| batch_results.delete(test) }
|
199
177
|
@logger.error("Did not run #{unrun.count}/#{tests.count}: #{unrun.join(' ')}") unless unrun.empty?
|
200
178
|
@logger.error("Extraneous runs (#{surfeit.count}): #{surfeit.join(' ')}") unless surfeit.empty?
|
@@ -235,7 +213,7 @@ module ParallelCucumber
|
|
235
213
|
@logger.warn("There was exception in on_batch_error hook #{exc.message} \n #{trace}")
|
236
214
|
end
|
237
215
|
|
238
|
-
return
|
216
|
+
return Helper::Cucumber.unknown_result(tests)
|
239
217
|
end
|
240
218
|
parse_results(test_state, tests)
|
241
219
|
ensure
|
@@ -294,18 +272,18 @@ module ParallelCucumber
|
|
294
272
|
def parse_results(f, tests)
|
295
273
|
unless File.file?(f)
|
296
274
|
@logger.error("Results file does not exist: #{f}")
|
297
|
-
return
|
275
|
+
return Helper::Cucumber.unknown_result(tests)
|
298
276
|
end
|
299
277
|
json_report = File.read(f)
|
300
278
|
if json_report.empty?
|
301
279
|
@logger.error("Results file is empty: #{f}")
|
302
|
-
return
|
280
|
+
return Helper::Cucumber.unknown_result(tests)
|
303
281
|
end
|
304
282
|
Helper::Cucumber.parse_json_report(json_report)
|
305
283
|
rescue => e
|
306
284
|
trace = e.backtrace.join("\n\t").sub("\n\t", ": #{$ERROR_INFO}#{e.class ? " (#{e.class})" : ''}\n\t")
|
307
285
|
@logger.error("Threw: JSON parse of results caused #{trace}")
|
308
|
-
|
286
|
+
Helper::Cucumber.unknown_result(tests)
|
309
287
|
end
|
310
288
|
end
|
311
289
|
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
module ParallelCucumber
|
2
|
+
class WorkerManager
|
3
|
+
def initialize(options, logger)
|
4
|
+
@options = options
|
5
|
+
@batch_size = options[:batch_size]
|
6
|
+
@logger = logger
|
7
|
+
@queue_connection_params = options[:queue_connection_params]
|
8
|
+
@backlog = ParallelCucumber::Helper::Queue.new(@queue_connection_params)
|
9
|
+
@queue_tracker = Tracker.new(@backlog)
|
10
|
+
@back_up_worker_size = options[:backup_worker_count]
|
11
|
+
@directed_queues = Hash.new do |hash, key|
|
12
|
+
hash[key] = ParallelCucumber::Helper::Queue.new(@queue_connection_params, "_#{key}")
|
13
|
+
end
|
14
|
+
@workers = {}
|
15
|
+
@unchecked_workers = Queue.new
|
16
|
+
@healthy_workers = Queue.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def start(number_of_workers)
|
20
|
+
create_workers(number_of_workers)
|
21
|
+
start_managing
|
22
|
+
start_workers
|
23
|
+
end
|
24
|
+
|
25
|
+
def kill
|
26
|
+
@current_thread.kill
|
27
|
+
end
|
28
|
+
|
29
|
+
def inform_healthy(worker)
|
30
|
+
@healthy_workers.enq(worker)
|
31
|
+
end
|
32
|
+
|
33
|
+
def inform_idle(worker)
|
34
|
+
@unchecked_workers.enq(worker)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def create_workers(number_of_workers)
|
40
|
+
number_of_workers.times do |index|
|
41
|
+
@workers["W#{index}"] =
|
42
|
+
ParallelCucumber::Worker.new(options: @options, index: index, stdout_logger: @logger, manager: self)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def start_managing
|
47
|
+
@current_thread = Thread.start do
|
48
|
+
loop do
|
49
|
+
if !@backlog.empty?
|
50
|
+
pre_check_unchecked_workers
|
51
|
+
give_job_to_healthy_worker
|
52
|
+
elsif any_worker_busy?
|
53
|
+
kill_surplus_workers
|
54
|
+
else
|
55
|
+
kill_all_workers
|
56
|
+
break
|
57
|
+
end
|
58
|
+
sleep 0.5
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def start_workers
|
64
|
+
indices = (0...@workers.size).to_a
|
65
|
+
@results = Parallel.map(indices.dup, in_threads: @workers.size,
|
66
|
+
finish: ->(_, ix, _) { @logger.synch { |l| l.info("Finished: #{ix} remaining: #{indices -= [ix]}") } }) do |index|
|
67
|
+
puts "Starting W#{index}"
|
68
|
+
@workers["W#{index}"].start(env_for_worker(@options[:env_variables], index))
|
69
|
+
end
|
70
|
+
@results.inject do |seed, result|
|
71
|
+
seed.merge(result) do |_key, oldval, newval|
|
72
|
+
(newval[:finish_time] > oldval[:finish_time]) ? newval : oldval
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def kill_all_workers
|
78
|
+
@logger.info('=== Killing All Workers')
|
79
|
+
@workers.values.each { |w| w.assign_job(Job.new(Job::DIE)) }
|
80
|
+
end
|
81
|
+
|
82
|
+
def kill_surplus_workers
|
83
|
+
until (@unchecked_workers.size + @healthy_workers.size) <= @back_up_worker_size
|
84
|
+
queue = !@unchecked_workers.empty? ? @unchecked_workers : @healthy_workers
|
85
|
+
worker = queue.pop(true)
|
86
|
+
@logger.info("Backup workers more than #{@back_up_worker_size}, killing #{worker}")
|
87
|
+
@workers[worker].assign_job(Job.new(Job::DIE))
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def pre_check_unchecked_workers
|
92
|
+
while !@unchecked_workers.empty? && worker = @unchecked_workers.pop(false)
|
93
|
+
@logger.info("=== #{worker} was asked precheck")
|
94
|
+
@workers[worker].assign_job(Job.new(Job::PRECHECK))
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def give_job_to_healthy_worker
|
99
|
+
return if @healthy_workers.empty?
|
100
|
+
|
101
|
+
worker_name = @healthy_workers.pop(true)
|
102
|
+
worker = @workers[worker_name]
|
103
|
+
batch = []
|
104
|
+
directed_queue = @directed_queues[worker.index]
|
105
|
+
@batch_size.times do
|
106
|
+
batch << (directed_queue.empty? ? @backlog : directed_queue).dequeue
|
107
|
+
end
|
108
|
+
batch.compact!
|
109
|
+
@logger.info("=== #{worker_name} was assigned #{batch.count} from the queue (#{@queue_tracker.status}): #{batch.join(' ')}")
|
110
|
+
worker.assign_job(Job.new(Job::RUN_TESTS, batch))
|
111
|
+
end
|
112
|
+
|
113
|
+
def any_worker_busy?
|
114
|
+
@workers.values.any?(&:busy_running_test?)
|
115
|
+
end
|
116
|
+
|
117
|
+
def env_for_worker(env_variables, worker_number)
|
118
|
+
env = env_variables.map do |k, v|
|
119
|
+
case v
|
120
|
+
when String, Numeric, TrueClass, FalseClass
|
121
|
+
[k, v]
|
122
|
+
when Array
|
123
|
+
[k, v[worker_number]]
|
124
|
+
when Hash
|
125
|
+
value = v[worker_number.to_s]
|
126
|
+
[k, value] unless value.nil?
|
127
|
+
when NilClass
|
128
|
+
else
|
129
|
+
raise("Don't know how to set '#{v}'<#{v.class}> to the environment variable '#{k}'")
|
130
|
+
end
|
131
|
+
end.compact.to_h
|
132
|
+
|
133
|
+
# Defaults, if absent in env. Shame 'merge' isn't something non-commutative like 'adopts/defaults'.
|
134
|
+
env = { TEST: 1, TEST_PROCESS_NUMBER: worker_number, WORKER_INDEX: worker_number }.merge(env)
|
135
|
+
|
136
|
+
# Overwrite this if it exists in env.
|
137
|
+
env.merge(PARALLEL_CUCUMBER_EXPORTS: env.keys.join(',')).map { |k, v| [k.to_s, v.to_s] }.to_h
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
class Tracker
|
142
|
+
def initialize(queue)
|
143
|
+
@backlog = queue
|
144
|
+
end
|
145
|
+
|
146
|
+
def status
|
147
|
+
queue_length = @backlog.length
|
148
|
+
now = Time.now
|
149
|
+
@full ||= queue_length
|
150
|
+
@start ||= now
|
151
|
+
completed = @full - queue_length
|
152
|
+
elapsed = now - @start
|
153
|
+
estimate = (completed == 0) ? '' : " #{(elapsed * @full / completed).to_i}s est"
|
154
|
+
"#{queue_length}/#{@full} left #{elapsed.to_i}s worker#{estimate}"
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel_cucumber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Bayandin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cucumber
|
@@ -101,11 +101,13 @@ files:
|
|
101
101
|
- lib/parallel_cucumber/helper/unittest/cucumber_test.rb
|
102
102
|
- lib/parallel_cucumber/helper/utils.rb
|
103
103
|
- lib/parallel_cucumber/hooks.rb
|
104
|
+
- lib/parallel_cucumber/job.rb
|
104
105
|
- lib/parallel_cucumber/logger.rb
|
105
106
|
- lib/parallel_cucumber/main.rb
|
106
107
|
- lib/parallel_cucumber/status.rb
|
107
108
|
- lib/parallel_cucumber/version.rb
|
108
109
|
- lib/parallel_cucumber/worker.rb
|
110
|
+
- lib/parallel_cucumber/worker_manager.rb
|
109
111
|
homepage: https://github.com/badoo/parallel_cucumber
|
110
112
|
licenses:
|
111
113
|
- MIT
|
@@ -125,8 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
127
|
- !ruby/object:Gem::Version
|
126
128
|
version: '0'
|
127
129
|
requirements: []
|
128
|
-
|
129
|
-
rubygems_version: 2.7.8
|
130
|
+
rubygems_version: 3.0.8
|
130
131
|
signing_key:
|
131
132
|
specification_version: 4
|
132
133
|
summary: Run cucumber in parallel
|