hydra 0.23.3 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/hydra/worker.rb CHANGED
@@ -19,8 +19,18 @@ module Hydra #:nodoc:
19
19
  @io = opts.fetch(:io) { raise "No IO Object" }
20
20
  @runners = []
21
21
  @listeners = []
22
+ @options = opts.fetch(:options) { "" }
22
23
 
23
24
  load_worker_initializer
25
+
26
+ @runner_event_listeners = Array(opts.fetch(:runner_listeners) { nil })
27
+ @runner_event_listeners.select{|l| l.is_a? String}.each do |l|
28
+ @runner_event_listeners.delete_at(@runner_event_listeners.index(l))
29
+ listener = eval(l)
30
+ @runner_event_listeners << listener if listener.is_a?(Hydra::RunnerListener::Abstract)
31
+ end
32
+ @runner_log_file = opts.fetch(:runner_log_file) { nil }
33
+
24
34
  boot_runners(opts.fetch(:runners) { 1 })
25
35
  @io.write(Hydra::Messages::Worker::WorkerBegin.new)
26
36
 
@@ -81,9 +91,10 @@ module Hydra #:nodoc:
81
91
  trace "Booting #{num_runners} Runners"
82
92
  num_runners.times do
83
93
  pipe = Hydra::Pipe.new
94
+
84
95
  child = SafeFork.fork do
85
96
  pipe.identify_as_child
86
- Hydra::Runner.new(:io => pipe, :verbose => @verbose)
97
+ Hydra::Runner.new(:io => pipe, :verbose => @verbose, :runner_listeners => @runner_event_listeners, :runner_log_file => @runner_log_file, :options => @options)
87
98
  end
88
99
  pipe.identify_as_parent
89
100
  @runners << { :pid => child, :io => pipe, :idle => false }
@@ -121,7 +132,7 @@ module Hydra #:nodoc:
121
132
  end
122
133
  rescue IOError => ex
123
134
  trace "Worker lost Master"
124
- Thread.exit
135
+ shutdown
125
136
  end
126
137
  end
127
138
  end
@@ -0,0 +1,2 @@
1
+ require '../test/fixtures/runner_listeners.rb'
2
+ require '../test/fixtures/master_listeners.rb'
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ 10000.times do
4
+ $stdout.write "A non-hydra message...\n"
5
+ $stdout.flush
6
+ end
7
+
8
+ $stdout.write "{:class=>Hydra::Messages::TestMessage, :text=>\"My message\"}\n"
9
+ $stdout.flush
@@ -0,0 +1,10 @@
1
+ module HydraExtension
2
+ module Listener
3
+ class WorkerBeganFlag < Hydra::Listener::Abstract
4
+ # Fired after runner processes have been started
5
+ def worker_begin(worker)
6
+ FileUtils.touch File.expand_path(File.join(Dir.consistent_tmpdir, 'worker_began_flag'))
7
+ end
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,23 @@
1
+ module HydraExtension
2
+ module RunnerListener
3
+ class RunnerBeginTest < Hydra::RunnerListener::Abstract
4
+ # Fired by the runner just before requesting the first file
5
+ def runner_begin( runner )
6
+ FileUtils.touch File.expand_path(File.join(Dir.consistent_tmpdir, 'alternate_hydra_test.txt'))
7
+ end
8
+ end
9
+
10
+ class RunnerEndTest < Hydra::RunnerListener::Abstract
11
+ # Fired by the runner just before requesting the first file
12
+ def runner_begin( runner )
13
+ FileUtils.touch File.expand_path(File.join(Dir.consistent_tmpdir, 'runner_began_flag')) #used to know when the runner is ready
14
+ end
15
+ # Fired by the runner just after stoping
16
+ def runner_end( runner )
17
+ # NOTE: do not use trace here
18
+ #runner.trace "Ending runner"
19
+ FileUtils.touch File.expand_path(File.join(Dir.consistent_tmpdir, 'alternate_hydra_test.txt'))
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,6 @@
1
+ ---
2
+ workers:
3
+ - type: ssh
4
+ connect: localhost
5
+ directory: /tmp
6
+ runners: 1
data/test/master_test.rb CHANGED
@@ -1,4 +1,6 @@
1
1
  require File.join(File.dirname(__FILE__), 'test_helper')
2
+ require File.join(File.dirname(__FILE__), 'fixtures', 'runner_listeners')
3
+ require File.join(File.dirname(__FILE__), 'fixtures', 'master_listeners')
2
4
 
3
5
  class MasterTest < Test::Unit::TestCase
4
6
  context "with a file to test and a destination to verify" do
@@ -115,8 +117,8 @@ class MasterTest < Test::Unit::TestCase
115
117
  :workers => [{
116
118
  :type => :ssh,
117
119
  :connect => 'localhost',
118
- :directory => File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
119
- :runners => 1
120
+ :directory => remote_dir_path,
121
+ :runners => 1
120
122
  }]
121
123
  )
122
124
  assert File.exists?(target_file)
@@ -178,4 +180,204 @@ class MasterTest < Test::Unit::TestCase
178
180
  assert !File.exists?(File.join(remote, 'test_b.rb')), "B was not deleted"
179
181
  end
180
182
  end
183
+
184
+ context "with a runner_end event" do
185
+ setup do
186
+ # avoid having other tests interfering with us
187
+ sleep(0.2)
188
+ FileUtils.rm_f(target_file)
189
+ FileUtils.rm_f(alternate_target_file)
190
+
191
+ @runner_began_flag = File.expand_path(File.join(Dir.consistent_tmpdir, 'runner_began_flag')) #used to know when the worker is ready
192
+ FileUtils.rm_f(@runner_began_flag)
193
+
194
+ @runner_listener = 'HydraExtension::RunnerListener::RunnerEndTest.new' # runner_end method that creates alternate_target_file
195
+ @master_listener = HydraExtension::Listener::WorkerBeganFlag.new #used to know when the runner is up
196
+ end
197
+
198
+ teardown do
199
+ FileUtils.rm_f(target_file)
200
+ FileUtils.rm_f(alternate_target_file)
201
+ end
202
+
203
+ context "running a local worker" do
204
+ should "run runner_end on successful termination" do
205
+ @pid = Process.fork do
206
+ Hydra::Master.new(
207
+ :files => [test_file] * 6,
208
+ :autosort => false,
209
+ :listeners => [@master_listener],
210
+ :runner_listeners => [@runner_listener],
211
+ :verbose => false
212
+ )
213
+ end
214
+ Process.waitpid @pid
215
+
216
+ assert_file_exists alternate_target_file
217
+ end
218
+
219
+ should "run runner_end after interruption signal" do
220
+ add_infinite_worker_begin_to @master_listener
221
+
222
+ capture_stderr do # redirect stderr
223
+ @pid = Process.fork do
224
+ Hydra::Master.new(
225
+ :files => [test_file],
226
+ :autosort => false,
227
+ :listeners => [@master_listener],
228
+ :runner_listeners => [@runner_listener],
229
+ :verbose => false
230
+ )
231
+ end
232
+ end
233
+ wait_for_runner_to_begin
234
+
235
+ Process.kill 'SIGINT', @pid
236
+ Process.waitpid @pid
237
+
238
+ assert_file_exists alternate_target_file
239
+ end
240
+ end
241
+
242
+ context "running a remote worker" do
243
+ setup do
244
+ copy_worker_init_file # this method has a protection to avoid erasing an existing worker_init_file
245
+ end
246
+
247
+ teardown do
248
+ FileUtils.rm_f(@remote_init_file) unless @protect_init_file
249
+ end
250
+
251
+ should "run runner_end on successful termination" do
252
+ capture_stderr do # redirect stderr
253
+ @pid = Process.fork do
254
+ Hydra::Master.new(
255
+ :files => [test_file],
256
+ :autosort => false,
257
+ :listeners => [@master_listener],
258
+ :runner_listeners => [@runner_listener],
259
+ :workers => [{
260
+ :type => :ssh,
261
+ :connect => 'localhost',
262
+ :directory => remote_dir_path,
263
+ :runners => 1
264
+ }],
265
+ :verbose => false
266
+ )
267
+ end
268
+ end
269
+ Process.waitpid @pid
270
+
271
+ assert_file_exists alternate_target_file
272
+ end
273
+ end
274
+ end
275
+
276
+ context "redirecting runner's output and errors" do
277
+ setup do
278
+ # avoid having other tests interfering with us
279
+ sleep(0.2)
280
+ FileUtils.rm_f(target_file)
281
+ FileUtils.rm_f(runner_log_file)
282
+ FileUtils.rm_f("#{remote_dir_path}/#{runner_log_file}")
283
+ end
284
+
285
+ teardown do
286
+ FileUtils.rm_f(target_file)
287
+ FileUtils.rm_f(runner_log_file)
288
+ FileUtils.rm_f("#{remote_dir_path}/#{runner_log_file}")
289
+ end
290
+
291
+ should "create a runner log file when usign local worker and passing a log file name" do
292
+ @pid = Process.fork do
293
+ Hydra::Master.new(
294
+ :files => [test_file],
295
+ :runner_log_file => runner_log_file,
296
+ :verbose => false
297
+ )
298
+ end
299
+ Process.waitpid @pid
300
+
301
+ assert_file_exists target_file # ensure the test was successfully ran
302
+ assert_file_exists runner_log_file
303
+ end
304
+
305
+ should "create a runner log file when usign remote worker and passing a log file name" do
306
+ @pid = Process.fork do
307
+ Hydra::Master.new(
308
+ :files => [test_file],
309
+ :workers => [{
310
+ :type => :ssh,
311
+ :connect => 'localhost',
312
+ :directory => remote_dir_path,
313
+ :runners => 1
314
+ }],
315
+ :verbose => false,
316
+ :runner_log_file => runner_log_file
317
+ )
318
+ end
319
+ Process.waitpid @pid
320
+
321
+ assert_file_exists target_file # ensure the test was successfully ran
322
+ assert_file_exists "#{remote_dir_path}/#{runner_log_file}"
323
+ end
324
+
325
+ should "create the default runner log file when passing an incorrect log file path" do
326
+ default_log_file = "#{remote_dir_path}/#{Hydra::Runner::DEFAULT_LOG_FILE}" # hydra-runner.log"
327
+ FileUtils.rm_f(default_log_file)
328
+
329
+ @pid = Process.fork do
330
+ Hydra::Master.new(
331
+ :files => [test_file],
332
+ :workers => [{
333
+ :type => :ssh,
334
+ :connect => 'localhost',
335
+ :directory => remote_dir_path,
336
+ :runners => 1
337
+ }],
338
+ :verbose => false,
339
+ :runner_log_file => 'invalid-dir/#{runner_log_file}'
340
+ )
341
+ end
342
+ Process.waitpid @pid
343
+
344
+ assert_file_exists target_file # ensure the test was successfully ran
345
+ assert_file_exists default_log_file #default log file
346
+ assert !File.exists?( "#{remote_dir_path}/#{runner_log_file}" )
347
+
348
+ FileUtils.rm_f(default_log_file)
349
+ end
350
+ end
351
+
352
+ private
353
+
354
+ def runner_log_file
355
+ "my-hydra-runner.log"
356
+ end
357
+
358
+ def add_infinite_worker_begin_to master_listener
359
+ class << master_listener
360
+ def worker_begin( worker )
361
+ super
362
+ sleep 1 while true #ensure the process doesn't finish before killing it
363
+ end
364
+ end
365
+ end
366
+
367
+ # this requires that a worker_begin listener creates a file named worker_began_flag in tmp directory
368
+ def wait_for_runner_to_begin
369
+ assert_file_exists @runner_began_flag
370
+ end
371
+
372
+ # with a protection to avoid erasing something important in lib
373
+ def copy_worker_init_file
374
+ @remote_init_file = "#{remote_dir_path}/#{File.basename( hydra_worker_init_file )}"
375
+ if File.exists?( @remote_init_file )
376
+ $stderr.puts "\nWARNING!!!: #{@remote_init_file} exits and this test needs to create a new file here. Make sure there is nothing inportant in that file and remove it before running this test\n\n"
377
+ @protect_init_file = true
378
+ exit
379
+ end
380
+ # copy the hydra_worker_init to the correct location
381
+ FileUtils.cp(hydra_worker_init_file, remote_dir_path)
382
+ end
181
383
  end
data/test/runner_test.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require File.join(File.dirname(__FILE__), 'test_helper')
2
+ require File.join(File.dirname(__FILE__), 'fixtures', 'runner_listeners')
2
3
 
3
4
  class RunnerTest < Test::Unit::TestCase
4
5
  context "with a file to test and a destination to verify" do
@@ -79,46 +80,91 @@ class RunnerTest < Test::Unit::TestCase
79
80
  # because of all the crap cucumber pulls in
80
81
  # we run this in a fork to not contaminate
81
82
  # the main test environment
82
- pid = Process.fork do
83
- runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
84
- runner.run_file(cucumber_feature_file)
85
- assert File.exists?(target_file)
86
- assert_equal "HYDRA", File.read(target_file)
87
-
88
- FileUtils.rm_f(target_file)
89
-
90
- runner.run_file(alternate_cucumber_feature_file)
91
- assert File.exists?(alternate_target_file)
92
- assert_equal "HYDRA", File.read(alternate_target_file)
93
- assert !File.exists?(target_file)
83
+ capture_stderr do # redirect stderr
84
+ pid = Process.fork do
85
+ runner = Hydra::Runner.new(:io => File.new('/dev/null', 'w'))
86
+ runner.run_file(cucumber_feature_file)
87
+ assert File.exists?(target_file)
88
+ assert_equal "HYDRA", File.read(target_file)
89
+
90
+ FileUtils.rm_f(target_file)
91
+
92
+ runner.run_file(alternate_cucumber_feature_file)
93
+ assert File.exists?(alternate_target_file)
94
+ assert_equal "HYDRA", File.read(alternate_target_file)
95
+ assert !File.exists?(target_file)
96
+ end
97
+ Process.wait pid
94
98
  end
95
- Process.wait pid
96
99
  end
97
100
 
98
101
  should "be able to run a runner over ssh" do
99
102
  ssh = Hydra::SSH.new(
100
- 'localhost',
103
+ 'localhost',
101
104
  File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')),
102
105
  "ruby -e \"require 'rubygems'; require 'hydra'; Hydra::Runner.new(:io => Hydra::Stdio.new, :verbose => true);\""
103
106
  )
104
107
  assert ssh.gets.is_a?(Hydra::Messages::Runner::RequestFile)
105
108
  ssh.write(Hydra::Messages::Worker::RunFile.new(:file => test_file))
106
-
109
+
107
110
  # grab its response. This makes us wait for it to finish
108
111
  echo = ssh.gets # get the ssh echo
109
112
  response = ssh.gets
110
113
 
111
114
  assert_equal Hydra::Messages::Runner::Results, response.class
112
-
115
+
113
116
  # tell it to shut down
114
117
  ssh.write(Hydra::Messages::Worker::Shutdown.new)
115
118
 
116
119
  ssh.close
117
-
120
+
118
121
  # ensure it ran
119
122
  assert File.exists?(target_file)
120
123
  assert_equal "HYDRA", File.read(target_file)
121
124
  end
125
+
126
+ context "using runner events" do
127
+ context "on successful termination" do
128
+ setup do
129
+ @pipe = Hydra::Pipe.new
130
+ @parent = Process.fork do
131
+ request_a_file_and_verify_completion(@pipe, test_file)
132
+ end
133
+ end
134
+
135
+ should "fire runner_begin event" do
136
+ run_the_runner(@pipe, [HydraExtension::RunnerListener::RunnerBeginTest.new] )
137
+ Process.wait(@parent)
138
+
139
+ # ensure runner_begin was fired
140
+ assert_file_exists alternate_target_file
141
+ end
142
+
143
+ should "fire runner_end event" do
144
+ run_the_runner(@pipe, [HydraExtension::RunnerListener::RunnerEndTest.new] )
145
+ Process.wait(@parent)
146
+
147
+ assert_file_exists alternate_target_file
148
+ end
149
+ end
150
+
151
+ should "fire runner_end event after losing communication with worker" do
152
+ pipe = Hydra::Pipe.new
153
+ parent = Process.fork do
154
+ pipe.identify_as_parent
155
+
156
+ # grab its response.
157
+ response = pipe.gets
158
+ pipe.close #this will be detected by the runner and it should call runner_end
159
+ end
160
+
161
+ run_the_runner(pipe, [HydraExtension::RunnerListener::RunnerEndTest.new] )
162
+ Process.wait(parent)
163
+
164
+ # ensure runner_end was fired
165
+ assert File.exists?( alternate_target_file )
166
+ end
167
+ end
122
168
  end
123
169
 
124
170
  module RunnerTestHelper
@@ -140,9 +186,9 @@ class RunnerTest < Test::Unit::TestCase
140
186
  assert_equal "HYDRA", File.read(target_file)
141
187
  end
142
188
 
143
- def run_the_runner(pipe)
189
+ def run_the_runner(pipe, listeners = [])
144
190
  pipe.identify_as_child
145
- Hydra::Runner.new(:io => pipe)
191
+ Hydra::Runner.new( :io => pipe, :runner_listeners => listeners )
146
192
  end
147
193
  end
148
194
  include RunnerTestHelper
data/test/ssh_test.rb CHANGED
@@ -11,4 +11,15 @@ class SSHTest < Test::Unit::TestCase
11
11
  assert_equal "Hello World", response.text
12
12
  ssh.close
13
13
  end
14
+
15
+ should "be able to handle a large number of non-Hydra console output" do
16
+ ssh = Hydra::SSH.new(
17
+ 'localhost', # connect to this machine
18
+ File.expand_path(File.join(File.dirname(__FILE__))), # move to the test directory
19
+ "ruby fixtures/many_outputs_to_console.rb"
20
+ )
21
+ response = ssh.gets
22
+ assert_equal "My message", response.text
23
+ ssh.close
24
+ end
14
25
  end