buffet 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/buffet +5 -0
- data/lib/buffet/cli.rb +39 -0
- data/lib/buffet/command_runner.rb +23 -0
- data/lib/buffet/master.rb +320 -0
- data/lib/buffet/project.rb +24 -0
- data/lib/buffet/runner.rb +155 -0
- data/lib/buffet/settings.rb +90 -0
- data/lib/buffet/slave.rb +48 -0
- data/lib/buffet/version.rb +3 -0
- data/lib/buffet.rb +64 -0
- data/support/buffet-worker +59 -0
- data/support/rspec1_formatter.rb +44 -0
- data/support/rspec2_formatter.rb +59 -0
- metadata +117 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 05b4680ad4e35179ea117b6d6208096ae463eb51
|
4
|
+
data.tar.gz: 69c35e59adc7f62883427b7eea842966b06f32cc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 549dd260454181a04fbc16e92ce13e238d18e28d2aa4ebeaf05a4727204e30804e5452518b065a21ff05ac8826608417669accd93696ea24b9bb4c23c08d594f
|
7
|
+
data.tar.gz: ca3c1037f6b62ba78e7cf6ad5e4101d1795f22502dfdb33f1cc870566d0eab503a7476e24148f3ca7d6e2db3800e809024a03a17b662585a2aef9df98ef5112c
|
data/bin/buffet
ADDED
data/lib/buffet/cli.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'buffet'
|
2
|
+
require 'optparse'
|
3
|
+
|
4
|
+
module Buffet
|
5
|
+
class CLI
|
6
|
+
def initialize args
|
7
|
+
opts = OptionParser.new do |opts|
|
8
|
+
opts.banner = "Usage: buffet [options] [spec-files]"
|
9
|
+
|
10
|
+
opts.on('-c', '--config CONFIG',
|
11
|
+
'Use the specified CONFIG file') do |config_file|
|
12
|
+
Settings.settings_file = File.expand_path(config_file)
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on('-p', '--project PROJECT',
|
16
|
+
'Use the specified PROJECT name') do |project_name|
|
17
|
+
Settings.project_name = project_name
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on('-l', '--log LOGFILE',
|
21
|
+
'Write to log to LOGFILE, instead of default buffet.log') do |log|
|
22
|
+
Settings.log_file = log
|
23
|
+
end
|
24
|
+
|
25
|
+
opts.on('-v', '--version', 'Show version') do
|
26
|
+
puts "#{opts.program_name} #{VERSION}"
|
27
|
+
exit
|
28
|
+
end
|
29
|
+
end.parse!(args)
|
30
|
+
|
31
|
+
specs = Buffet.extract_specs_from(opts.empty? ? ['spec'] : opts)
|
32
|
+
|
33
|
+
runner = Runner.new
|
34
|
+
runner.run specs
|
35
|
+
|
36
|
+
exit 1 if runner.failed?
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'wopen3'
|
3
|
+
|
4
|
+
module Buffet
|
5
|
+
class CommandRunner
|
6
|
+
def initialize logger = Logger.new(STDOUT)
|
7
|
+
@logger = logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def run *command
|
11
|
+
start_time = Time.now
|
12
|
+
result = Wopen3.system *command
|
13
|
+
end_time = Time.now
|
14
|
+
@logger.info "\n" +
|
15
|
+
"command: #{command.join ' '}\n" +
|
16
|
+
"time: #{end_time - start_time}\n" +
|
17
|
+
"status: #{result.status}\n" +
|
18
|
+
"stdout:\n#{result.stdout}\n" +
|
19
|
+
"stderr:\n#{result.stderr}"
|
20
|
+
result
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,320 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
require 'drb'
|
3
|
+
require 'thread'
|
4
|
+
require 'socket'
|
5
|
+
|
6
|
+
module Buffet
|
7
|
+
class Master
|
8
|
+
attr_reader :failures, :stats, :slave_exceptions, :spurious_failures
|
9
|
+
|
10
|
+
def initialize project, slaves, specs, listener
|
11
|
+
@project = project
|
12
|
+
@slaves = slaves
|
13
|
+
@stats = { :examples => 0, :failures => 0, :pending => 0, :spurious_failures => 0 }
|
14
|
+
@slaves_stats = Hash[@slaves.map do |slave|
|
15
|
+
[slave.user_at_host, stats.dup.merge!(:slave => slave, :specs => [])]
|
16
|
+
end]
|
17
|
+
@slave_exceptions = {}
|
18
|
+
@max_slave_prepare_failures = [Settings.allowed_slave_prepare_failures,
|
19
|
+
@slaves.count - 1].min
|
20
|
+
@stats[:slaves] = @slaves_stats
|
21
|
+
@lock = Mutex.new
|
22
|
+
@condition = ConditionVariable.new
|
23
|
+
@failures = []
|
24
|
+
@spurious_failures = []
|
25
|
+
@specs = order_specs(specs)
|
26
|
+
@listener = listener
|
27
|
+
@halt_exception = nil
|
28
|
+
|
29
|
+
# How many times a particular spec file was queued
|
30
|
+
@spec_queue_count = Hash.new { |h, k| h[k] = 0 }
|
31
|
+
|
32
|
+
# Need this so when we get back an example result from a slave we can know
|
33
|
+
# which spec it is from. This prevents the wrong file from being reported
|
34
|
+
# in the case of shared examples.
|
35
|
+
@current_spec_for_slave = {}
|
36
|
+
|
37
|
+
# Store spec results on per spec per line basis
|
38
|
+
@spec_results = Hash.new { |h, k| h[k] = [] }
|
39
|
+
end
|
40
|
+
|
41
|
+
def order_specs(specs)
|
42
|
+
specs.map do |spec|
|
43
|
+
[`wc -l #{spec}`.to_i, spec]
|
44
|
+
end.sort.reverse.map(&:last)
|
45
|
+
end
|
46
|
+
|
47
|
+
def run
|
48
|
+
start_service
|
49
|
+
|
50
|
+
@stats[:total_time] = Benchmark.measure do
|
51
|
+
@threads = @slaves.map do |slave|
|
52
|
+
Thread.new do
|
53
|
+
run_worker slave
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
wait_for_workers
|
58
|
+
end.real
|
59
|
+
|
60
|
+
stop_service
|
61
|
+
|
62
|
+
process_spec_results
|
63
|
+
end
|
64
|
+
|
65
|
+
def run_worker slave
|
66
|
+
time = Benchmark.measure do
|
67
|
+
begin
|
68
|
+
prepare_slave slave
|
69
|
+
rescue CommandError => ex
|
70
|
+
slave_prepare_failed slave, ex
|
71
|
+
rescue Exception => ex
|
72
|
+
stop_run ex
|
73
|
+
else
|
74
|
+
begin
|
75
|
+
run_slave slave
|
76
|
+
rescue Exception => ex
|
77
|
+
stop_run ex
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end.real
|
81
|
+
|
82
|
+
@lock.synchronize do
|
83
|
+
@slaves_stats[slave.name][:total_time] = time
|
84
|
+
@condition.signal # Tell master this slave is finished
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def wait_for_workers
|
89
|
+
@threads.count.times do
|
90
|
+
@lock.synchronize do
|
91
|
+
@condition.wait(@lock)
|
92
|
+
|
93
|
+
raise @halt_exception if @halt_exception
|
94
|
+
|
95
|
+
if slave_exceptions.count > @max_slave_prepare_failures
|
96
|
+
raise 'Exceeded maximum number of allowed slave prepare failures'
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def stop_run ex
|
103
|
+
@lock.synchronize do
|
104
|
+
@halt_exception = ex
|
105
|
+
@condition.signal # Alert master
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def next_file_for(slave_name, previous_spec)
|
110
|
+
file = nil
|
111
|
+
|
112
|
+
@lock.synchronize do
|
113
|
+
if rerun_spec?(previous_spec)
|
114
|
+
@spec_queue_count[previous_spec] += 1
|
115
|
+
return previous_spec
|
116
|
+
else
|
117
|
+
if file = @specs.shift
|
118
|
+
@spec_queue_count[file] += 1
|
119
|
+
@current_spec_for_slave[slave_name] = file
|
120
|
+
|
121
|
+
@slaves_stats[slave_name][:specs] << file
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
if file
|
127
|
+
slave = nil
|
128
|
+
@lock.synchronize do
|
129
|
+
slave = @slaves_stats[slave_name][:slave]
|
130
|
+
end
|
131
|
+
@listener.spec_taken(slave, file)
|
132
|
+
end
|
133
|
+
|
134
|
+
file
|
135
|
+
end
|
136
|
+
|
137
|
+
def slave_prepare_failed slave, ex
|
138
|
+
@listener.slave_prepare_failed slave, ex
|
139
|
+
|
140
|
+
@lock.synchronize do
|
141
|
+
slave_exceptions[slave.name] = ex
|
142
|
+
|
143
|
+
@condition.signal # Alert master this slave failed
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def example_passed(slave_name, details)
|
148
|
+
@lock.synchronize do
|
149
|
+
@spec_results[@current_spec_for_slave[slave_name]] << details
|
150
|
+
end
|
151
|
+
|
152
|
+
@listener.example_passed
|
153
|
+
end
|
154
|
+
|
155
|
+
def example_failed(slave_name, details)
|
156
|
+
@lock.synchronize do
|
157
|
+
@spec_results[@current_spec_for_slave[slave_name]] << details
|
158
|
+
end
|
159
|
+
|
160
|
+
@listener.example_failed
|
161
|
+
end
|
162
|
+
|
163
|
+
def example_pending slave_name, details
|
164
|
+
@lock.synchronize do
|
165
|
+
@stats[:examples] += 1
|
166
|
+
@stats[:pending] += 1
|
167
|
+
@slaves_stats[slave_name][:pending] += 1
|
168
|
+
end
|
169
|
+
|
170
|
+
@listener.example_pending
|
171
|
+
end
|
172
|
+
|
173
|
+
private
|
174
|
+
|
175
|
+
def server_uri
|
176
|
+
@drb_server.uri
|
177
|
+
end
|
178
|
+
|
179
|
+
def start_service
|
180
|
+
@drb_server = DRb.start_service("druby://#{ip}:0", self)
|
181
|
+
end
|
182
|
+
|
183
|
+
def stop_service
|
184
|
+
DRb.stop_service
|
185
|
+
end
|
186
|
+
|
187
|
+
def ip
|
188
|
+
result = Buffet.run! 'host `hostname -s`'
|
189
|
+
result.stdout.chomp.match(/((\d+\.){3}\d+)/)[1]
|
190
|
+
end
|
191
|
+
|
192
|
+
def prepare_slave slave
|
193
|
+
time = Benchmark.measure do
|
194
|
+
@project.sync_to slave
|
195
|
+
|
196
|
+
if Settings.has_prepare_script?
|
197
|
+
slave.execute_in_project "#{Settings.prepare_script} #{Buffet.user} #{@project.name}"
|
198
|
+
end
|
199
|
+
|
200
|
+
# Copy support files so they can be run on the remote machine
|
201
|
+
slave.scp File.dirname(__FILE__) + '/../../support',
|
202
|
+
@project.support_dir_on_slave, :recurse => true
|
203
|
+
end.real
|
204
|
+
@lock.synchronize { @slaves_stats[slave.name][:prepare_time] = time }
|
205
|
+
|
206
|
+
@listener.slave_prepared slave
|
207
|
+
end
|
208
|
+
|
209
|
+
def run_slave slave
|
210
|
+
time = Benchmark.measure do
|
211
|
+
slave.execute_in_project([
|
212
|
+
Settings.worker_command,
|
213
|
+
server_uri,
|
214
|
+
slave.user_at_host,
|
215
|
+
Settings.framework,
|
216
|
+
].join(' '))
|
217
|
+
end.real
|
218
|
+
|
219
|
+
@lock.synchronize { @slaves_stats[slave.name][:test_time] = time }
|
220
|
+
|
221
|
+
@listener.slave_finished slave
|
222
|
+
end
|
223
|
+
|
224
|
+
def rerun_spec?(spec)
|
225
|
+
spec && example_failed_last_run?(spec) && unconfirmed_failures(spec) > 0
|
226
|
+
end
|
227
|
+
|
228
|
+
def example_count(spec)
|
229
|
+
@spec_results[spec].count / @spec_queue_count[spec]
|
230
|
+
end
|
231
|
+
|
232
|
+
def example_failed_last_run?(spec)
|
233
|
+
(1..example_count(spec)).any? do |i|
|
234
|
+
@spec_results[spec][-i][:status] == :failed
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
def confirmed_failures(spec)
|
239
|
+
failure_counts_for_spec(spec).count do |failure_count|
|
240
|
+
failure_count >= Settings.failure_threshold
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def unconfirmed_failures(spec)
|
245
|
+
failure_counts_for_spec(spec).count do |failure_count|
|
246
|
+
(1...Settings.failure_threshold).member?(failure_count)
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
# Returns an array of failure counts over all spec runs for each example
|
251
|
+
def failure_counts_for_spec(spec)
|
252
|
+
example_result_list = @spec_results[spec]
|
253
|
+
example_count = example_count(spec)
|
254
|
+
|
255
|
+
(0...example_count).map do |i|
|
256
|
+
example_result_list.select.
|
257
|
+
with_index { |_, j| j % example_count == i }.
|
258
|
+
count { |example_result| example_result[:status] == :failed }
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
def process_spec_results
|
263
|
+
@spec_results.each do |spec, example_results|
|
264
|
+
next if example_results.empty? # Can legitimately happen when spec has no examples
|
265
|
+
|
266
|
+
slave_name = example_results.last[:slave_name]
|
267
|
+
|
268
|
+
spec_examples_count = example_count(spec)
|
269
|
+
@stats[:examples] += spec_examples_count
|
270
|
+
@slaves_stats[slave_name][:examples] += spec_examples_count
|
271
|
+
|
272
|
+
confirmed_failures = confirmed_failures(spec)
|
273
|
+
@stats[:failures] += confirmed_failures
|
274
|
+
@slaves_stats[slave_name][:failures] += confirmed_failures
|
275
|
+
@failures += confirmed_failure_results(spec)
|
276
|
+
|
277
|
+
spurious_failures = unconfirmed_failures(spec)
|
278
|
+
@stats[:spurious_failures] += spurious_failures
|
279
|
+
@slaves_stats[slave_name][:spurious_failures] += spurious_failures
|
280
|
+
@spurious_failures += spurious_failure_results(spec)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
def confirmed_failure_results(spec)
|
285
|
+
failures = []
|
286
|
+
|
287
|
+
example_count = example_count(spec)
|
288
|
+
|
289
|
+
failure_counts_for_spec(spec).each_with_index do |count, index|
|
290
|
+
if count >= Settings.failure_threshold
|
291
|
+
# Find first failure for spec
|
292
|
+
failures << @spec_results[spec].
|
293
|
+
select.
|
294
|
+
with_index { |_, i| i % example_count == index }.
|
295
|
+
find { |example_result| example_result[:status] == :failed }
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
failures
|
300
|
+
end
|
301
|
+
|
302
|
+
def spurious_failure_results(spec)
|
303
|
+
spurious_failures = []
|
304
|
+
|
305
|
+
example_count = example_count(spec)
|
306
|
+
|
307
|
+
failure_counts_for_spec(spec).each_with_index do |count, index|
|
308
|
+
if (1...Settings.failure_threshold).member?(count)
|
309
|
+
# Find first failure for spec
|
310
|
+
spurious_failures << @spec_results[spec].
|
311
|
+
select.
|
312
|
+
with_index { |_, i| i % example_count == index }.
|
313
|
+
find { |example_result| example_result[:status] == :failed }
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
spurious_failures
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Buffet
|
2
|
+
class Project
|
3
|
+
attr_accessor :name
|
4
|
+
attr_reader :directory
|
5
|
+
|
6
|
+
def initialize directory
|
7
|
+
@name = File.basename directory
|
8
|
+
@directory = File.expand_path directory
|
9
|
+
end
|
10
|
+
|
11
|
+
def directory_on_slave
|
12
|
+
"#{Buffet.workspace_dir}/#{name}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def support_dir_on_slave
|
16
|
+
"#{directory_on_slave}/.buffet"
|
17
|
+
end
|
18
|
+
|
19
|
+
def sync_to slave
|
20
|
+
slave.execute "mkdir -p #{directory_on_slave}"
|
21
|
+
slave.rsync directory + '/', directory_on_slave
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
require 'buffet'
|
2
|
+
require 'colorize'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Buffet
|
6
|
+
class Runner
|
7
|
+
def initialize
|
8
|
+
@project = Settings.project
|
9
|
+
end
|
10
|
+
|
11
|
+
def run specs = nil
|
12
|
+
@specs = specs
|
13
|
+
raise 'No specs found' if @specs.empty?
|
14
|
+
|
15
|
+
@slaves = Settings.slaves
|
16
|
+
raise 'No slaves defined in settings.yml' if @slaves.empty?
|
17
|
+
|
18
|
+
Buffet.logger.info "Starting Buffet test run"
|
19
|
+
puts "Running Buffet..."
|
20
|
+
|
21
|
+
run_tests
|
22
|
+
display_results
|
23
|
+
end
|
24
|
+
|
25
|
+
def slave_prepared slave
|
26
|
+
Buffet.logger.info "#{slave.name} prepared"
|
27
|
+
end
|
28
|
+
|
29
|
+
def slave_prepare_failed slave, exception
|
30
|
+
Buffet.logger.warn "#{slave.name} preparation failed: #{exception}"
|
31
|
+
end
|
32
|
+
|
33
|
+
def spec_taken slave, spec_file
|
34
|
+
Buffet.logger.info "#{slave.name} took #{spec_file}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def example_passed
|
38
|
+
print '.'.green
|
39
|
+
STDOUT.flush
|
40
|
+
end
|
41
|
+
|
42
|
+
def example_failed
|
43
|
+
print 'F'.red
|
44
|
+
STDOUT.flush
|
45
|
+
end
|
46
|
+
|
47
|
+
def example_pending
|
48
|
+
print '*'.yellow
|
49
|
+
STDOUT.flush
|
50
|
+
end
|
51
|
+
|
52
|
+
def slave_finished slave
|
53
|
+
Buffet.logger.info "#{slave.name} finished"
|
54
|
+
gather_junit slave if !!Settings['gather_junit']
|
55
|
+
end
|
56
|
+
|
57
|
+
def failed?
|
58
|
+
@master.failures.any? || no_examples_run?
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def run_tests
|
64
|
+
@master = Master.new @project, @slaves, @specs, self
|
65
|
+
@master.run
|
66
|
+
end
|
67
|
+
|
68
|
+
def display_results
|
69
|
+
results = []
|
70
|
+
results << "\n"
|
71
|
+
|
72
|
+
results << "Total Examples: #{@master.stats[:examples]}"
|
73
|
+
results << "Total Pending: #{@master.stats[:pending]}"
|
74
|
+
results << "Total Failures: #{@master.stats[:failures]}"
|
75
|
+
results << "Total Spurious Failures: #{@master.stats[:spurious_failures]}"
|
76
|
+
results << "Buffet consumed in #{@master.stats[:total_time]} seconds"
|
77
|
+
|
78
|
+
if @master.failures.any?
|
79
|
+
results << '' << 'SPEC FAILURES:'.red
|
80
|
+
results << ('=' * 80).red
|
81
|
+
results << @master.failures.map do |failure|
|
82
|
+
example_details(failure)
|
83
|
+
end
|
84
|
+
elsif @master.spurious_failures.any?
|
85
|
+
results << '' << 'SPURIOUS FAILURES:'.yellow
|
86
|
+
results << ('-' * 80).yellow
|
87
|
+
results << @master.spurious_failures.map do |spurious_failure|
|
88
|
+
example_details(spurious_failure)
|
89
|
+
end
|
90
|
+
elsif no_examples_run?
|
91
|
+
results << '' << 'No examples were run!'.red
|
92
|
+
end
|
93
|
+
|
94
|
+
results << '' << 'SLAVE STATS:'
|
95
|
+
results << ('-' * 80)
|
96
|
+
@master.stats[:slaves].each do |slave_name, slave_stats|
|
97
|
+
results << "#{slave_name}:"
|
98
|
+
|
99
|
+
if @master.slave_exceptions[slave_name]
|
100
|
+
results << "\tFailed to prepare"
|
101
|
+
next
|
102
|
+
end
|
103
|
+
|
104
|
+
slave_stats.each do |key, value|
|
105
|
+
results << "\t#{key}: #{value}" unless [:slave, :specs].include?(key)
|
106
|
+
end
|
107
|
+
|
108
|
+
if slave_stats[:failures] > 0
|
109
|
+
results << "\tSpec order (use to help reproduce spurious failures):"
|
110
|
+
results << "\t #{slave_stats[:specs].join("\n\t ")}"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
unless @master.slave_exceptions.empty?
|
115
|
+
results << '' << 'SLAVE PREPARATION_FAILURES'.yellow
|
116
|
+
results << ('-' * 80).yellow
|
117
|
+
results << @master.slave_exceptions.map do |slave, ex|
|
118
|
+
"#{slave}:\n".yellow +
|
119
|
+
"#{ex}" + (ex.backtrace ? ex.backtrace.join("\n") : 'No backtrace')
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
puts results
|
124
|
+
end
|
125
|
+
|
126
|
+
def example_details(details)
|
127
|
+
"#{details[:description]}\n".red +
|
128
|
+
"Slave: #{details[:slave_name]}\n" +
|
129
|
+
"Location: #{details[:location]}\n" +
|
130
|
+
"#{details[:message]}\n".yellow +
|
131
|
+
"#{details[:backtrace]}\n"
|
132
|
+
end
|
133
|
+
|
134
|
+
def no_examples_run?
|
135
|
+
@master.stats[:examples] - @master.stats[:pending] == 0
|
136
|
+
end
|
137
|
+
|
138
|
+
def gather_junit slave
|
139
|
+
# Jenkins tells us where we are running
|
140
|
+
workspace = ENV['WORKSPACE'] || './'
|
141
|
+
FileUtils.mkdir_p File.join(workspace, 'reports')
|
142
|
+
|
143
|
+
# Copy the junit results from the slave into reports/foo@bar/
|
144
|
+
# Configure jenkins to read reports/**/*.xml
|
145
|
+
Buffet.run! *%W[
|
146
|
+
rsync -aqz --delete
|
147
|
+
-e ssh
|
148
|
+
#{slave.user_at_host}:#{slave.project.directory_on_slave}/spec/reports/
|
149
|
+
#{workspace}/reports/#{slave.name}/
|
150
|
+
]
|
151
|
+
rescue CommandError
|
152
|
+
Buffet.logger.warn "Failed to collect junit report from #{slave.name}"
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Buffet
|
4
|
+
class Settings
|
5
|
+
DEFAULT_LOG_FILE = 'buffet.log'
|
6
|
+
DEFAULT_SETTINGS_FILE = 'buffet.yml'
|
7
|
+
DEFAULT_PREPARE_SCRIPT = 'bin/before-buffet-run'
|
8
|
+
DEFAULT_EXCLUDE_FILTER_FILE = '.buffet-exclude-filter'
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def settings_file=(settings_file)
|
12
|
+
@settings_file = settings_file
|
13
|
+
reset!
|
14
|
+
end
|
15
|
+
|
16
|
+
def settings_file
|
17
|
+
@settings_file || DEFAULT_SETTINGS_FILE
|
18
|
+
end
|
19
|
+
|
20
|
+
def [](name)
|
21
|
+
@settings ||= load_file(settings_file)
|
22
|
+
@settings[name]
|
23
|
+
end
|
24
|
+
|
25
|
+
def slaves
|
26
|
+
@slaves ||= self['slaves'].map do |slave_hash|
|
27
|
+
Slave.new slave_hash['user'], slave_hash['host'], project
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def allowed_slave_prepare_failures
|
32
|
+
self['allowed_slave_prepare_failures'] || 0
|
33
|
+
end
|
34
|
+
|
35
|
+
def worker_command
|
36
|
+
self['worker_command'] || '.buffet/buffet-worker'
|
37
|
+
end
|
38
|
+
|
39
|
+
def log_file=(log)
|
40
|
+
@log_file = log
|
41
|
+
end
|
42
|
+
|
43
|
+
def log_file
|
44
|
+
@log_file || self['log_file'] || DEFAULT_LOG_FILE
|
45
|
+
end
|
46
|
+
|
47
|
+
def project_name=(project_name)
|
48
|
+
project.name = project_name
|
49
|
+
end
|
50
|
+
|
51
|
+
def project
|
52
|
+
@project ||= Project.new Dir.pwd
|
53
|
+
end
|
54
|
+
|
55
|
+
def framework
|
56
|
+
self['framework'].upcase || 'RSPEC1'
|
57
|
+
end
|
58
|
+
|
59
|
+
def prepare_script
|
60
|
+
self['prepare_script'] || DEFAULT_PREPARE_SCRIPT
|
61
|
+
end
|
62
|
+
|
63
|
+
def has_prepare_script?
|
64
|
+
self['prepare_script'] || File.exist?(DEFAULT_PREPARE_SCRIPT)
|
65
|
+
end
|
66
|
+
|
67
|
+
def exclude_filter_file
|
68
|
+
self['exclude_filter_file'] || DEFAULT_EXCLUDE_FILTER_FILE
|
69
|
+
end
|
70
|
+
|
71
|
+
def has_exclude_filter_file?
|
72
|
+
self['exclude_filter_file'] || File.exist?(DEFAULT_EXCLUDE_FILTER_FILE)
|
73
|
+
end
|
74
|
+
|
75
|
+
def failure_threshold
|
76
|
+
self['failure_threshold'] || 2
|
77
|
+
end
|
78
|
+
|
79
|
+
def reset!
|
80
|
+
@settings = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def load_file file
|
86
|
+
@settings = YAML.load_file file
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
data/lib/buffet/slave.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Buffet
|
2
|
+
class Slave
|
3
|
+
attr_reader :user, :host, :project
|
4
|
+
|
5
|
+
def initialize user, host, project
|
6
|
+
@user = user
|
7
|
+
@host = host
|
8
|
+
@project = project
|
9
|
+
end
|
10
|
+
|
11
|
+
def rsync src, dest
|
12
|
+
Buffet.run! 'rsync', '-aqz', '--delete',
|
13
|
+
'--delete-excluded', rsync_exclude_flags,
|
14
|
+
'-e', 'ssh', src, "#{user_at_host}:#{dest}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def scp src, dest, options = {}
|
18
|
+
args = [src, "#{user_at_host}:#{dest}"]
|
19
|
+
args.unshift '-r' if options[:recurse]
|
20
|
+
Buffet.run! 'scp', *args
|
21
|
+
end
|
22
|
+
|
23
|
+
def execute_in_project command
|
24
|
+
execute "cd #{@project.directory_on_slave} && #{command}"
|
25
|
+
end
|
26
|
+
|
27
|
+
def execute command
|
28
|
+
Buffet.run! 'ssh', "#{user_at_host}", command
|
29
|
+
end
|
30
|
+
|
31
|
+
def name
|
32
|
+
user_at_host
|
33
|
+
end
|
34
|
+
|
35
|
+
def user_at_host
|
36
|
+
"#{@user}@#{@host}"
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def rsync_exclude_flags
|
42
|
+
if Settings.has_exclude_filter_file?
|
43
|
+
exclude_flags = "--exclude-from=#{Settings.exclude_filter_file}"
|
44
|
+
end
|
45
|
+
exclude_flags || ''
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/buffet.rb
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'find'
|
3
|
+
require 'logger'
|
4
|
+
require 'pathname'
|
5
|
+
|
6
|
+
module Buffet
|
7
|
+
autoload :CommandRunner, 'buffet/command_runner'
|
8
|
+
autoload :Master, 'buffet/master'
|
9
|
+
autoload :Project, 'buffet/project'
|
10
|
+
autoload :Runner, 'buffet/runner'
|
11
|
+
autoload :Settings, 'buffet/settings'
|
12
|
+
autoload :Slave, 'buffet/slave'
|
13
|
+
|
14
|
+
class CommandError < StandardError; end
|
15
|
+
|
16
|
+
def self.log_dir
|
17
|
+
@log_dir ||= Pathname.new(ENV['HOME']) + '.buffet/log'
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.log_file
|
21
|
+
Settings.log_file
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.logger
|
25
|
+
@logger ||= begin
|
26
|
+
FileUtils.mkdir_p log_dir
|
27
|
+
Logger.new log_dir + log_file
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.runner
|
32
|
+
@runner ||= CommandRunner.new logger
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.run! *command
|
36
|
+
result = runner.run *command
|
37
|
+
unless result.success?
|
38
|
+
message = "`#{command.join(' ')}` exited with non-zero status: #{result.status}"
|
39
|
+
message += "\nSTDOUT: #{result.stdout}" unless result.stdout.empty?
|
40
|
+
message += "\nSTDERR: #{result.stderr}" unless result.stderr.empty?
|
41
|
+
raise CommandError.new(message)
|
42
|
+
end
|
43
|
+
result
|
44
|
+
end
|
45
|
+
|
46
|
+
# Given a set of files/directories, return all spec files contained
|
47
|
+
def self.extract_specs_from files
|
48
|
+
specs = []
|
49
|
+
files.each do |spec_file|
|
50
|
+
Find.find(spec_file) do |f|
|
51
|
+
specs << f if f.match /_spec\.rb$/
|
52
|
+
end
|
53
|
+
end
|
54
|
+
specs.uniq
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.workspace_dir
|
58
|
+
".buffet/workspaces/#{user}" # Relative to home directory
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.user
|
62
|
+
@user ||= `whoami`.chomp
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'drb'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
if File.exist? 'Gemfile'
|
7
|
+
require 'rubygems'
|
8
|
+
require 'bundler/setup'
|
9
|
+
end
|
10
|
+
|
11
|
+
# NOTE: ARGV is used by the spec runner. If I leave the drb server address
|
12
|
+
# in ARGV, rspec will think it's an argument to the test runner.
|
13
|
+
buffet_server = DRbObject.new_with_uri(ARGV.shift)
|
14
|
+
slave_name = ARGV.shift
|
15
|
+
framework = ARGV.shift
|
16
|
+
|
17
|
+
FileUtils.mkdir_p('./tmp')
|
18
|
+
|
19
|
+
if framework == 'RSPEC1'
|
20
|
+
require 'spec'
|
21
|
+
require 'spec/runner/command_line'
|
22
|
+
require File.expand_path('rspec1_formatter', File.dirname(__FILE__))
|
23
|
+
|
24
|
+
Spec::Runner::Formatter::AugmentedTextFormatter.configure buffet_server, slave_name
|
25
|
+
|
26
|
+
while file = buffet_server.next_file_for(slave_name)
|
27
|
+
# RSpec1 closes stderr/out after each run, so we reopen them each time
|
28
|
+
outlog = File.open('./tmp/buffet.out.log', 'a')
|
29
|
+
errlog = File.open('./tmp/buffet.error.log', 'a')
|
30
|
+
|
31
|
+
Spec::Runner::CommandLine.run(
|
32
|
+
Spec::Runner::OptionParser.parse(
|
33
|
+
['--format', 'Spec::Runner::Formatter::AugmentedTextFormatter', file],
|
34
|
+
errlog,
|
35
|
+
outlog
|
36
|
+
)
|
37
|
+
)
|
38
|
+
end
|
39
|
+
else
|
40
|
+
require 'rspec'
|
41
|
+
require File.expand_path('rspec2_formatter', File.dirname(__FILE__))
|
42
|
+
|
43
|
+
begin
|
44
|
+
require 'ci/reporter/rspec'
|
45
|
+
use_ci_reporter = true
|
46
|
+
rescue LoadError
|
47
|
+
end
|
48
|
+
|
49
|
+
RSpec::Core::Formatters::AugmentedTextFormatter.configure buffet_server, slave_name
|
50
|
+
RSpec::Core::Runner.disable_autorun!
|
51
|
+
rspec_opts = ['--format', 'RSpec::Core::Formatters::AugmentedTextFormatter']
|
52
|
+
rspec_opts += ['--format', 'CI::Reporter::RSpec'] if use_ci_reporter
|
53
|
+
|
54
|
+
while file = buffet_server.next_file_for(slave_name, file)
|
55
|
+
RSpec::Core::CommandLine.new(rspec_opts.dup << file).
|
56
|
+
run($stderr, $stdout)
|
57
|
+
RSpec.world.example_groups.clear
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'spec/runner/formatter/base_text_formatter'
|
2
|
+
|
3
|
+
module Spec
|
4
|
+
module Runner
|
5
|
+
module Formatter
|
6
|
+
class AugmentedTextFormatter < BaseTextFormatter
|
7
|
+
def self.configure buffet_server, slave_name
|
8
|
+
@@buffet_server = buffet_server
|
9
|
+
@@slave_name = slave_name
|
10
|
+
end
|
11
|
+
|
12
|
+
def example_passed example_proxy
|
13
|
+
super
|
14
|
+
@@buffet_server.example_passed(@@slave_name, {
|
15
|
+
:description => example_proxy.description,
|
16
|
+
:location => example_proxy.location,
|
17
|
+
:slave_name => @@slave_name,
|
18
|
+
})
|
19
|
+
end
|
20
|
+
|
21
|
+
def example_failed example_proxy, counter, failure
|
22
|
+
super
|
23
|
+
@@buffet_server.example_failed(@@slave_name, {
|
24
|
+
:backtrace => failure.exception.backtrace.join("\n"),
|
25
|
+
:description => failure.header,
|
26
|
+
:location => example_proxy.location,
|
27
|
+
:message => failure.exception.message,
|
28
|
+
:slave_name => @@slave_name,
|
29
|
+
})
|
30
|
+
end
|
31
|
+
|
32
|
+
def example_pending example, message, deprecated_pending_location=nil
|
33
|
+
super
|
34
|
+
@@buffet_server.example_pending(@@slave_name, {
|
35
|
+
:description => example.description,
|
36
|
+
:location => example.location,
|
37
|
+
:message => message,
|
38
|
+
:slave_name => @@slave_name,
|
39
|
+
})
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'rspec/core/formatters/base_text_formatter'
|
2
|
+
|
3
|
+
module RSpec
|
4
|
+
module Core
|
5
|
+
module Formatters
|
6
|
+
class AugmentedTextFormatter < BaseTextFormatter
|
7
|
+
def initialize(output)
|
8
|
+
super(output)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.configure buffet_server, slave_name
|
12
|
+
@@buffet_server = buffet_server
|
13
|
+
@@slave_name = slave_name
|
14
|
+
end
|
15
|
+
|
16
|
+
def example_passed example
|
17
|
+
super
|
18
|
+
|
19
|
+
@@buffet_server.example_passed(@@slave_name, {
|
20
|
+
:description => example.description,
|
21
|
+
:location => example.location,
|
22
|
+
:status => :passed,
|
23
|
+
:slave_name => @@slave_name,
|
24
|
+
})
|
25
|
+
end
|
26
|
+
|
27
|
+
def example_failed example
|
28
|
+
super
|
29
|
+
exception = example.metadata[:execution_result][:exception]
|
30
|
+
backtrace = format_backtrace(exception.backtrace, example).join("\n")
|
31
|
+
|
32
|
+
location = example.location
|
33
|
+
if shared_group = find_shared_group(example)
|
34
|
+
location += "\nShared example group called from " +
|
35
|
+
backtrace_line(shared_group.metadata[:example_group][:location])
|
36
|
+
end
|
37
|
+
|
38
|
+
@@buffet_server.example_failed(@@slave_name, {
|
39
|
+
:description => example.description,
|
40
|
+
:backtrace => backtrace,
|
41
|
+
:message => exception.message,
|
42
|
+
:location => location,
|
43
|
+
:status => :failed,
|
44
|
+
:slave_name => @@slave_name,
|
45
|
+
})
|
46
|
+
end
|
47
|
+
|
48
|
+
def example_pending example
|
49
|
+
super
|
50
|
+
@@buffet_server.example_pending(@@slave_name, {
|
51
|
+
:description => example.description,
|
52
|
+
:location => example.location,
|
53
|
+
:slave_name => @@slave_name,
|
54
|
+
})
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
metadata
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: buffet
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.3.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Causes Engineering
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: colorize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: wopen3
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mkdtemp
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Buffet distributes RSpec test cases over multiple machines.
|
70
|
+
email:
|
71
|
+
- eng@causes.com
|
72
|
+
- grant@causes.com
|
73
|
+
- shane@causes.com
|
74
|
+
executables:
|
75
|
+
- buffet
|
76
|
+
extensions: []
|
77
|
+
extra_rdoc_files: []
|
78
|
+
files:
|
79
|
+
- lib/buffet.rb
|
80
|
+
- lib/buffet/cli.rb
|
81
|
+
- lib/buffet/command_runner.rb
|
82
|
+
- lib/buffet/master.rb
|
83
|
+
- lib/buffet/project.rb
|
84
|
+
- lib/buffet/runner.rb
|
85
|
+
- lib/buffet/settings.rb
|
86
|
+
- lib/buffet/slave.rb
|
87
|
+
- lib/buffet/version.rb
|
88
|
+
- support/buffet-worker
|
89
|
+
- support/rspec1_formatter.rb
|
90
|
+
- support/rspec2_formatter.rb
|
91
|
+
- bin/buffet
|
92
|
+
homepage: http://github.com/causes/buffet
|
93
|
+
licenses:
|
94
|
+
- MIT
|
95
|
+
metadata: {}
|
96
|
+
post_install_message:
|
97
|
+
rdoc_options: []
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - '>='
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
requirements: []
|
111
|
+
rubyforge_project:
|
112
|
+
rubygems_version: 2.0.2
|
113
|
+
signing_key:
|
114
|
+
specification_version: 4
|
115
|
+
summary: Distributed testing framework for Ruby, Rails and RSpec
|
116
|
+
test_files: []
|
117
|
+
has_rdoc:
|