hydra 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/TODO +14 -52
- data/VERSION +1 -1
- data/hydra.gemspec +6 -2
- data/lib/hydra.rb +1 -0
- data/lib/hydra/master.rb +75 -0
- data/lib/hydra/message.rb +1 -0
- data/lib/hydra/message/master_messages.rb +19 -0
- data/lib/hydra/message/runner_messages.rb +7 -15
- data/lib/hydra/message/worker_messages.rb +29 -9
- data/lib/hydra/messaging_io.rb +4 -5
- data/lib/hydra/pipe.rb +6 -0
- data/lib/hydra/runner.rb +77 -15
- data/lib/hydra/worker.rb +59 -28
- data/test/fixtures/write_file.rb +1 -1
- data/test/master_test.rb +21 -0
- data/test/runner_test.rb +7 -13
- data/test/test_helper.rb +7 -0
- data/test/worker_test.rb +10 -17
- metadata +6 -2
data/TODO
CHANGED
@@ -1,54 +1,16 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
# out = pipes[c][1]
|
5
|
-
# runners << TestRunner.new(in, out)
|
6
|
-
# end
|
7
|
-
#
|
8
|
-
# files = [ ... ]
|
9
|
-
# results = []
|
10
|
-
#
|
11
|
-
# runners.each do |r|
|
12
|
-
# Thread.new do
|
13
|
-
# while !files.empty?
|
14
|
-
# results << r.run_file(files.pop)
|
15
|
-
# end
|
16
|
-
# r.shutdown
|
17
|
-
# end
|
18
|
-
# end
|
19
|
-
#
|
20
|
-
# puts results.join("\n")
|
21
|
-
#
|
1
|
+
IO selection configuration for master
|
2
|
+
- allow pipe setup
|
3
|
+
- allow ssh setup
|
22
4
|
|
5
|
+
YML configuration
|
6
|
+
|
7
|
+
v0.6.0
|
8
|
+
|
9
|
+
multitest backwards compatible
|
10
|
+
|
11
|
+
v0.7.0
|
12
|
+
|
13
|
+
???
|
14
|
+
|
15
|
+
v1.0.0
|
23
16
|
|
24
|
-
# Master
|
25
|
-
# boot up workers
|
26
|
-
# listen for worker messages
|
27
|
-
# add worker messages to message queue
|
28
|
-
# process message queue
|
29
|
-
# "reply" to a message allows sending a message back down to worker
|
30
|
-
#
|
31
|
-
# When worker asks for file but no files left, send shutdown message to worker
|
32
|
-
# when worker connection breaks, end thread
|
33
|
-
# wait on all threads
|
34
|
-
# when all threads are done, all workers must be done
|
35
|
-
#
|
36
|
-
#
|
37
|
-
# Worker
|
38
|
-
# boot up runners
|
39
|
-
# listen for runner messages
|
40
|
-
# add runner messages to message queue
|
41
|
-
# process message queue
|
42
|
-
# "reply" to a message allows sending message back down to runner
|
43
|
-
#
|
44
|
-
# when a runner asks for file but master responds with shutdown, mark self
|
45
|
-
# as terminated, shut down runners. Any runner that asks for a file is
|
46
|
-
# auto-terminated
|
47
|
-
# wait for runner threads to finish
|
48
|
-
# then exit, breaking master connection
|
49
|
-
#
|
50
|
-
# Runner
|
51
|
-
# when booted, ask for a file
|
52
|
-
# then process messages on the queue
|
53
|
-
# when it's a file, run it and send a results message
|
54
|
-
# when it's a shutdown, break main loop
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.5.0
|
data/hydra.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{hydra}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.5.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Nick Gauthier"]
|
12
|
-
s.date = %q{2010-01-
|
12
|
+
s.date = %q{2010-01-29}
|
13
13
|
s.description = %q{Spread your tests over multiple machines to test your code faster.}
|
14
14
|
s.email = %q{nick@smartlogicsolutions.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -27,7 +27,9 @@ Gem::Specification.new do |s|
|
|
27
27
|
"VERSION",
|
28
28
|
"hydra.gemspec",
|
29
29
|
"lib/hydra.rb",
|
30
|
+
"lib/hydra/master.rb",
|
30
31
|
"lib/hydra/message.rb",
|
32
|
+
"lib/hydra/message/master_messages.rb",
|
31
33
|
"lib/hydra/message/runner_messages.rb",
|
32
34
|
"lib/hydra/message/worker_messages.rb",
|
33
35
|
"lib/hydra/messaging_io.rb",
|
@@ -38,6 +40,7 @@ Gem::Specification.new do |s|
|
|
38
40
|
"test/fixtures/assert_true.rb",
|
39
41
|
"test/fixtures/echo_the_dolphin.rb",
|
40
42
|
"test/fixtures/write_file.rb",
|
43
|
+
"test/master_test.rb",
|
41
44
|
"test/message_test.rb",
|
42
45
|
"test/pipe_test.rb",
|
43
46
|
"test/runner_test.rb",
|
@@ -57,6 +60,7 @@ Gem::Specification.new do |s|
|
|
57
60
|
"test/fixtures/write_file.rb",
|
58
61
|
"test/fixtures/assert_true.rb",
|
59
62
|
"test/fixtures/echo_the_dolphin.rb",
|
63
|
+
"test/master_test.rb",
|
60
64
|
"test/worker_test.rb",
|
61
65
|
"test/runner_test.rb",
|
62
66
|
"test/pipe_test.rb"
|
data/lib/hydra.rb
CHANGED
data/lib/hydra/master.rb
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
module Hydra #:nodoc:
|
2
|
+
# Hydra class responsible for delegate work down to workers.
|
3
|
+
#
|
4
|
+
# The Master is run once for any given testing session.
|
5
|
+
class Master
|
6
|
+
include Hydra::Messages::Master
|
7
|
+
# Create a new Master
|
8
|
+
#
|
9
|
+
# Options:
|
10
|
+
# * :files
|
11
|
+
# * An array of test files to be run. These should be relative paths from
|
12
|
+
# the root of the project, since they may be run on different machines
|
13
|
+
# which may have different paths.
|
14
|
+
# * :workers
|
15
|
+
# * An array of hashes. Each hash should be the configuration options
|
16
|
+
# for a worker.
|
17
|
+
def initialize(opts = { })
|
18
|
+
@files = opts.fetch(:files) { [] }
|
19
|
+
@workers = []
|
20
|
+
@listeners = []
|
21
|
+
boot_workers(opts.fetch(:workers) { [ {:runners => 1} ] } )
|
22
|
+
process_messages
|
23
|
+
end
|
24
|
+
|
25
|
+
# Message handling
|
26
|
+
|
27
|
+
# Send a file down to a worker. If there are no more files, this will shut the
|
28
|
+
# worker down.
|
29
|
+
def send_file(worker)
|
30
|
+
f = @files.pop
|
31
|
+
if f
|
32
|
+
worker[:io].write(RunFile.new(:file => f))
|
33
|
+
else
|
34
|
+
worker[:io].write(Shutdown.new)
|
35
|
+
Thread.exit
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def boot_workers(workers)
|
42
|
+
workers.each do |worker|
|
43
|
+
pipe = Hydra::Pipe.new
|
44
|
+
child = Process.fork do
|
45
|
+
pipe.identify_as_child
|
46
|
+
Hydra::Worker.new(:io => pipe, :runners => worker[:runners])
|
47
|
+
end
|
48
|
+
pipe.identify_as_parent
|
49
|
+
@workers << { :pid => child, :io => pipe, :idle => false }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def process_messages
|
54
|
+
Thread.abort_on_exception = true
|
55
|
+
|
56
|
+
@workers.each do |worker|
|
57
|
+
@listeners << Thread.new do
|
58
|
+
while true
|
59
|
+
begin
|
60
|
+
message = worker[:io].gets
|
61
|
+
message.handle(self, worker) if message
|
62
|
+
rescue IOError => ex
|
63
|
+
$stderr.write "Master lost Worker [#{worker.inspect}]\n"
|
64
|
+
worker[:io].close
|
65
|
+
@workers.delete(worker)
|
66
|
+
Thread.exit
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
@listeners.each{|l| l.join}
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
data/lib/hydra/message.rb
CHANGED
@@ -0,0 +1,19 @@
|
|
1
|
+
module Hydra #:nodoc:
|
2
|
+
module Messages #:nodoc:
|
3
|
+
module Master #:nodoc:
|
4
|
+
# Message telling a worker to delegate a file to a runner
|
5
|
+
class RunFile < Hydra::Messages::Worker::RunFile
|
6
|
+
def handle(worker) #:nodoc:
|
7
|
+
worker.delegate_file(self)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Message telling the worker to shut down.
|
12
|
+
class Shutdown < Hydra::Messages::Worker::Shutdown
|
13
|
+
def handle(worker) #:nodoc:
|
14
|
+
worker.shutdown
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -8,20 +8,11 @@ module Hydra #:nodoc:
|
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
11
|
-
# Message telling the Runner to run a file
|
12
|
-
class RunFile < Hydra::Message
|
13
|
-
attr_accessor :file
|
14
|
-
def serialize #:nodoc:
|
15
|
-
super(:file => @file)
|
16
|
-
end
|
17
|
-
def handle(runner) #:nodoc:
|
18
|
-
runner.run_file(@file)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
11
|
# Message for the Runner to respond with its results
|
23
12
|
class Results < Hydra::Message
|
13
|
+
# The output from running the test
|
24
14
|
attr_accessor :output
|
15
|
+
# The file that was run
|
25
16
|
attr_accessor :file
|
26
17
|
def serialize #:nodoc:
|
27
18
|
super(:output => @output, :file => @file)
|
@@ -31,10 +22,11 @@ module Hydra #:nodoc:
|
|
31
22
|
end
|
32
23
|
end
|
33
24
|
|
34
|
-
# Message to
|
35
|
-
class
|
36
|
-
def handle(runner) #:nodoc:
|
37
|
-
|
25
|
+
# Message a runner sends to a worker to verify the connection
|
26
|
+
class Ping < Hydra::Message
|
27
|
+
def handle(worker, runner) #:nodoc:
|
28
|
+
# We don't do anything to handle a ping. It's just to test
|
29
|
+
# the connectivity of the IO
|
38
30
|
end
|
39
31
|
end
|
40
32
|
end
|
@@ -1,25 +1,45 @@
|
|
1
1
|
module Hydra #:nodoc:
|
2
2
|
module Messages #:nodoc:
|
3
3
|
module Worker #:nodoc:
|
4
|
-
# Message indicating that a
|
4
|
+
# Message indicating that a worker needs a file to delegate to a runner
|
5
5
|
class RequestFile < Hydra::Message
|
6
|
+
def handle(master, worker) #:nodoc:
|
7
|
+
master.send_file(worker)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Message telling the Runner to run a file
|
12
|
+
class RunFile < Hydra::Message
|
13
|
+
# The file that should be run
|
14
|
+
attr_accessor :file
|
15
|
+
def serialize #:nodoc:
|
16
|
+
super(:file => @file)
|
17
|
+
end
|
18
|
+
def handle(runner) #:nodoc:
|
19
|
+
runner.run_file(@file)
|
20
|
+
end
|
6
21
|
end
|
7
22
|
|
8
|
-
# Message
|
9
|
-
class
|
10
|
-
def handle(
|
11
|
-
|
23
|
+
# Message to tell the Runner to shut down
|
24
|
+
class Shutdown < Hydra::Message
|
25
|
+
def handle(runner) #:nodoc:
|
26
|
+
runner.stop
|
12
27
|
end
|
13
28
|
end
|
14
29
|
|
15
30
|
# Message relaying the results of a worker up to the master
|
16
31
|
class Results < Hydra::Messages::Runner::Results
|
32
|
+
def handle(master, worker) #:nodoc:
|
33
|
+
$stdout.write output
|
34
|
+
master.send_file(worker)
|
35
|
+
end
|
17
36
|
end
|
18
37
|
|
19
|
-
# Message
|
20
|
-
class
|
21
|
-
def handle(worker)
|
22
|
-
|
38
|
+
# Message a worker sends to a master to verify the connection
|
39
|
+
class Ping < Hydra::Message
|
40
|
+
def handle(master, worker) #:nodoc:
|
41
|
+
# We don't do anything to handle a ping. It's just to test
|
42
|
+
# the connectivity of the IO
|
23
43
|
end
|
24
44
|
end
|
25
45
|
end
|
data/lib/hydra/messaging_io.rb
CHANGED
@@ -20,11 +20,9 @@ module Hydra #:nodoc:
|
|
20
20
|
def write(message)
|
21
21
|
raise IOError unless @writer
|
22
22
|
raise UnprocessableMessage unless message.is_a?(Hydra::Message)
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
raise IOError
|
27
|
-
end
|
23
|
+
@writer.write(message.serialize+"\n")
|
24
|
+
rescue Errno::EPIPE
|
25
|
+
raise IOError
|
28
26
|
end
|
29
27
|
|
30
28
|
# Closes the IO object.
|
@@ -37,6 +35,7 @@ module Hydra #:nodoc:
|
|
37
35
|
# For example, if you tried to write a string, it would fail,
|
38
36
|
# because the string is not a message.
|
39
37
|
class UnprocessableMessage < RuntimeError
|
38
|
+
# Custom error message
|
40
39
|
attr_accessor :message
|
41
40
|
end
|
42
41
|
end
|
data/lib/hydra/pipe.rb
CHANGED
data/lib/hydra/runner.rb
CHANGED
@@ -1,32 +1,94 @@
|
|
1
1
|
module Hydra #:nodoc:
|
2
|
-
# Hydra class responsible for running test files
|
2
|
+
# Hydra class responsible for running test files.
|
3
|
+
#
|
4
|
+
# The Runner is never run directly by a user. Runners are created by a
|
5
|
+
# Worker to run test files.
|
6
|
+
#
|
7
|
+
# The general convention is to have one Runner for each logical processor
|
8
|
+
# of a machine.
|
3
9
|
class Runner
|
10
|
+
include Hydra::Messages::Runner
|
4
11
|
# Boot up a runner. It takes an IO object (generally a pipe from its
|
5
12
|
# parent) to send it messages on which files to execute.
|
6
|
-
def initialize(
|
7
|
-
@io = io
|
8
|
-
@
|
9
|
-
process_messages
|
10
|
-
end
|
13
|
+
def initialize(opts = {})
|
14
|
+
@io = opts.fetch(:io) { raise "No IO Object" }
|
15
|
+
@verbose = opts.fetch(:verbose) { false }
|
11
16
|
|
12
|
-
|
13
|
-
|
14
|
-
@
|
15
|
-
|
16
|
-
message = @io.gets
|
17
|
-
message.handle(self) if message
|
18
|
-
end
|
17
|
+
Test::Unit.run = true
|
18
|
+
|
19
|
+
@io.write RequestFile.new
|
20
|
+
process_messages
|
19
21
|
end
|
20
22
|
|
21
23
|
# Run a test file and report the results
|
22
24
|
def run_file(file)
|
23
|
-
|
24
|
-
|
25
|
+
require file
|
26
|
+
output = []
|
27
|
+
@result = Test::Unit::TestResult.new
|
28
|
+
@result.add_listener(Test::Unit::TestResult::FAULT) do |value|
|
29
|
+
output << value
|
30
|
+
end
|
31
|
+
|
32
|
+
klasses = Runner.find_classes_in_file(file)
|
33
|
+
begin
|
34
|
+
klasses.each{|klass| klass.suite.run(@result){|status, name| ;}}
|
35
|
+
rescue => ex
|
36
|
+
output << ex.to_s
|
37
|
+
end
|
38
|
+
|
39
|
+
output << '.' if output.empty?
|
40
|
+
|
41
|
+
@io.write Results.new(:output => output.join("\n"), :file => file)
|
25
42
|
end
|
26
43
|
|
27
44
|
# Stop running
|
28
45
|
def stop
|
29
46
|
@running = false
|
30
47
|
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
# The runner will continually read messages and handle them.
|
52
|
+
def process_messages
|
53
|
+
$stdout.write "RUNNER| Processing Messages\n" if @verbose
|
54
|
+
@running = true
|
55
|
+
while @running
|
56
|
+
begin
|
57
|
+
message = @io.gets
|
58
|
+
if message
|
59
|
+
$stdout.write "RUNNER| Received message from worker\n" if @verbose
|
60
|
+
$stdout.write " | #{message.inspect}\n" if @verbose
|
61
|
+
message.handle(self)
|
62
|
+
else
|
63
|
+
@io.write Ping.new
|
64
|
+
end
|
65
|
+
rescue IOError => ex
|
66
|
+
$stderr.write "Runner lost Worker\n" if @verbose
|
67
|
+
@running = false
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.find_classes_in_file(f)
|
73
|
+
code = ""
|
74
|
+
File.open(f) {|buffer| code = buffer.read}
|
75
|
+
matches = code.scan(/class\s+([\S]+)/)
|
76
|
+
klasses = matches.collect do |c|
|
77
|
+
begin
|
78
|
+
if c.first.respond_to? :constantize
|
79
|
+
c.first.constantize
|
80
|
+
else
|
81
|
+
eval(c.first)
|
82
|
+
end
|
83
|
+
rescue NameError
|
84
|
+
# $stderr.write "Could not load [#{c.first}] from [#{f}]\n"
|
85
|
+
nil
|
86
|
+
rescue SyntaxError
|
87
|
+
# $stderr.write "Could not load [#{c.first}] from [#{f}]\n"
|
88
|
+
nil
|
89
|
+
end
|
90
|
+
end
|
91
|
+
return klasses.select{|k| k.respond_to? 'suite'}
|
92
|
+
end
|
31
93
|
end
|
32
94
|
end
|
data/lib/hydra/worker.rb
CHANGED
@@ -1,15 +1,25 @@
|
|
1
1
|
module Hydra #:nodoc:
|
2
2
|
# Hydra class responsible to dispatching runners and communicating with the master.
|
3
|
+
#
|
4
|
+
# The Worker is never run directly by a user. Workers are created by a
|
5
|
+
# Master to delegate to Runners.
|
6
|
+
#
|
7
|
+
# The general convention is to have one Worker per machine on a distributed
|
8
|
+
# network.
|
3
9
|
class Worker
|
10
|
+
include Hydra::Messages::Worker
|
4
11
|
# Create a new worker.
|
5
12
|
# * io: The IO object to use to communicate with the master
|
6
13
|
# * num_runners: The number of runners to launch
|
7
|
-
def initialize(
|
8
|
-
@
|
14
|
+
def initialize(opts = {})
|
15
|
+
@verbose = opts.fetch(:verbose) { false }
|
16
|
+
@io = opts.fetch(:io) { raise "No IO Object" }
|
9
17
|
@runners = []
|
10
18
|
@listeners = []
|
11
|
-
|
19
|
+
|
20
|
+
boot_runners(opts.fetch(:runners) { 1 })
|
12
21
|
process_messages
|
22
|
+
|
13
23
|
@runners.each{|r| Process.wait r[:pid] }
|
14
24
|
end
|
15
25
|
|
@@ -19,93 +29,114 @@ module Hydra #:nodoc:
|
|
19
29
|
# When a runner wants a file, it hits this method with a message.
|
20
30
|
# Then the worker bubbles the file request up to the master.
|
21
31
|
def request_file(message, runner)
|
22
|
-
@io.write(
|
32
|
+
@io.write(RequestFile.new)
|
23
33
|
runner[:idle] = true
|
24
34
|
end
|
25
35
|
|
26
36
|
# When the master sends a file down to the worker, it hits this
|
27
37
|
# method. Then the worker delegates the file down to a runner.
|
28
38
|
def delegate_file(message)
|
29
|
-
|
30
|
-
|
31
|
-
|
39
|
+
runner = idle_runner
|
40
|
+
runner[:idle] = false
|
41
|
+
runner[:io].write(RunFile.new(eval(message.serialize)))
|
32
42
|
end
|
33
43
|
|
34
44
|
# When a runner finishes, it sends the results up to the worker. Then the
|
35
45
|
# worker sends the results up to the master.
|
36
|
-
# TODO: when we relay results, it should trigger a RunFile or Shutdown from
|
37
|
-
# the master implicitly
|
38
46
|
def relay_results(message, runner)
|
39
47
|
runner[:idle] = true
|
40
|
-
@io.write(
|
48
|
+
@io.write(Results.new(eval(message.serialize)))
|
41
49
|
end
|
42
50
|
|
43
51
|
# When a master issues a shutdown order, it hits this method, which causes
|
44
52
|
# the worker to send shutdown messages to its runners.
|
45
|
-
# TODO: implement a ShutdownComplete message, so that we can kill the
|
46
|
-
# processes if necessary.
|
47
53
|
def shutdown
|
48
54
|
@running = false
|
55
|
+
$stdout.write "WORKER| Notifying #{@runners.size} Runners of Shutdown\n" if @verbose
|
49
56
|
@runners.each do |r|
|
50
|
-
|
57
|
+
$stdout.write "WORKER| Sending Shutdown to Runner\n" if @verbose
|
58
|
+
$stdout.write " | #{r.inspect}\n" if @verbose
|
59
|
+
r[:io].write(Shutdown.new)
|
51
60
|
end
|
61
|
+
Thread.exit
|
52
62
|
end
|
53
63
|
|
54
64
|
private
|
55
65
|
|
56
66
|
def boot_runners(num_runners) #:nodoc:
|
67
|
+
$stdout.write "WORKER| Booting #{num_runners} Runners\n" if @verbose
|
57
68
|
num_runners.times do
|
58
69
|
pipe = Hydra::Pipe.new
|
59
70
|
child = Process.fork do
|
60
71
|
pipe.identify_as_child
|
61
|
-
Hydra::Runner.new(pipe)
|
72
|
+
Hydra::Runner.new(:io => pipe, :verbose => @verbose)
|
62
73
|
end
|
63
74
|
pipe.identify_as_parent
|
64
75
|
@runners << { :pid => child, :io => pipe, :idle => false }
|
65
76
|
end
|
77
|
+
$stdout.write "WORKER| #{@runners.size} Runners booted\n" if @verbose
|
66
78
|
end
|
67
79
|
|
68
80
|
# Continuously process messages
|
69
81
|
def process_messages #:nodoc:
|
82
|
+
$stdout.write "WORKER| Processing Messages\n" if @verbose
|
70
83
|
@running = true
|
71
84
|
|
72
|
-
# Abort the worker if one of the runners has an exception
|
73
|
-
# TODO: catch this exception, return a dying message to the master
|
74
|
-
# then shutdown
|
75
85
|
Thread.abort_on_exception = true
|
76
86
|
|
77
|
-
|
87
|
+
process_messages_from_master
|
88
|
+
process_messages_from_runners
|
89
|
+
|
90
|
+
@listeners.each{|l| l.join }
|
91
|
+
@io.close
|
92
|
+
$stdout.write "WORKER| Done processing messages\n" if @verbose
|
93
|
+
end
|
94
|
+
|
95
|
+
def process_messages_from_master
|
78
96
|
@listeners << Thread.new do
|
79
97
|
while @running
|
80
|
-
|
81
|
-
|
98
|
+
begin
|
99
|
+
message = @io.gets
|
100
|
+
if message
|
101
|
+
$stdout.write "WORKER| Received Message from Master\n" if @verbose
|
102
|
+
$stdout.write " | #{message.inspect}\n" if @verbose
|
103
|
+
message.handle(self)
|
104
|
+
else
|
105
|
+
@io.write Ping.new
|
106
|
+
end
|
107
|
+
rescue IOError => ex
|
108
|
+
$stderr.write "Worker lost Master\n" if @verbose
|
109
|
+
Thread.exit
|
110
|
+
end
|
82
111
|
end
|
83
112
|
end
|
113
|
+
end
|
84
114
|
|
85
|
-
|
86
|
-
# so we can reference them when we deal with their messages
|
115
|
+
def process_messages_from_runners
|
87
116
|
@runners.each do |r|
|
88
117
|
@listeners << Thread.new do
|
89
118
|
while @running
|
90
119
|
begin
|
91
120
|
message = r[:io].gets
|
92
|
-
|
121
|
+
if message
|
122
|
+
$stdout.write "WORKER| Received Message from Runner\n" if @verbose
|
123
|
+
$stdout.write " | #{message.inspect}\n" if @verbose
|
124
|
+
message.handle(self, r)
|
125
|
+
end
|
93
126
|
rescue IOError => ex
|
94
|
-
|
95
|
-
|
96
|
-
# not @running anymore
|
127
|
+
$stderr.write "Worker lost Runner [#{r.inspect}]\n"
|
128
|
+
Thread.exit
|
97
129
|
end
|
98
130
|
end
|
99
131
|
end
|
100
132
|
end
|
101
|
-
@listeners.each{|l| l.join }
|
102
133
|
end
|
103
134
|
|
104
135
|
# Get the next idle runner
|
105
136
|
def idle_runner #:nodoc:
|
106
137
|
idle_r = nil
|
107
138
|
while idle_r.nil?
|
108
|
-
idle_r = @runners.detect{|
|
139
|
+
idle_r = @runners.detect{|runner| runner[:idle]}
|
109
140
|
sleep(1)
|
110
141
|
end
|
111
142
|
return idle_r
|
data/test/fixtures/write_file.rb
CHANGED
data/test/master_test.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'test_helper')
|
2
|
+
|
3
|
+
class MasterTest < Test::Unit::TestCase
|
4
|
+
context "with a file to test and a destination to verify" do
|
5
|
+
setup do
|
6
|
+
FileUtils.rm_f(target_file)
|
7
|
+
end
|
8
|
+
|
9
|
+
teardown do
|
10
|
+
FileUtils.rm_f(target_file)
|
11
|
+
end
|
12
|
+
|
13
|
+
should "run a test" do
|
14
|
+
m = Hydra::Master.new({
|
15
|
+
:files => Array(test_file)
|
16
|
+
})
|
17
|
+
assert File.exists?(target_file)
|
18
|
+
assert_equal "HYDRA", File.read(target_file)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/test/runner_test.rb
CHANGED
@@ -1,16 +1,13 @@
|
|
1
1
|
require File.join(File.dirname(__FILE__), 'test_helper')
|
2
2
|
|
3
|
-
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
|
4
|
-
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
|
5
|
-
|
6
3
|
class RunnerTest < Test::Unit::TestCase
|
7
4
|
context "with a file to test and a destination to verify" do
|
8
5
|
setup do
|
9
|
-
FileUtils.rm_f(
|
6
|
+
FileUtils.rm_f(target_file)
|
10
7
|
end
|
11
8
|
|
12
9
|
teardown do
|
13
|
-
FileUtils.rm_f(
|
10
|
+
FileUtils.rm_f(target_file)
|
14
11
|
end
|
15
12
|
|
16
13
|
|
@@ -44,25 +41,22 @@ class RunnerTest < Test::Unit::TestCase
|
|
44
41
|
|
45
42
|
# make sure it asks for a file, then give it one
|
46
43
|
assert pipe.gets.is_a?(Hydra::Messages::Runner::RequestFile)
|
47
|
-
pipe.write(Hydra::Messages::
|
44
|
+
pipe.write(Hydra::Messages::Worker::RunFile.new(:file => test_file))
|
48
45
|
|
49
46
|
# grab its response. This makes us wait for it to finish
|
50
47
|
response = pipe.gets
|
51
48
|
|
52
49
|
# tell it to shut down
|
53
|
-
pipe.write(Hydra::Messages::
|
50
|
+
pipe.write(Hydra::Messages::Worker::Shutdown.new)
|
54
51
|
|
55
52
|
# ensure it ran
|
56
|
-
assert File.exists?(
|
57
|
-
assert_equal "HYDRA", File.read(
|
58
|
-
|
59
|
-
pipe.close
|
53
|
+
assert File.exists?(target_file)
|
54
|
+
assert_equal "HYDRA", File.read(target_file)
|
60
55
|
end
|
61
56
|
|
62
57
|
def run_the_runner(pipe)
|
63
58
|
pipe.identify_as_child
|
64
|
-
Hydra::Runner.new(pipe)
|
65
|
-
pipe.close
|
59
|
+
Hydra::Runner.new({:io => pipe})
|
66
60
|
end
|
67
61
|
end
|
68
62
|
include RunnerTestHelper
|
data/test/test_helper.rb
CHANGED
@@ -8,6 +8,13 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
8
8
|
require 'hydra'
|
9
9
|
|
10
10
|
class Test::Unit::TestCase
|
11
|
+
def target_file
|
12
|
+
File.join(Dir.tmpdir, 'hydra_test.txt')
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_file
|
16
|
+
File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
|
17
|
+
end
|
11
18
|
end
|
12
19
|
|
13
20
|
module Hydra #:nodoc:
|
data/test/worker_test.rb
CHANGED
@@ -1,32 +1,28 @@
|
|
1
1
|
require File.join(File.dirname(__FILE__), 'test_helper')
|
2
2
|
|
3
|
-
TARGET = File.join(Dir.tmpdir, 'hydra_test.txt')
|
4
|
-
TESTFILE = File.join(File.dirname(__FILE__), 'fixtures', 'write_file.rb')
|
5
|
-
|
6
3
|
class WorkerTest < Test::Unit::TestCase
|
7
4
|
context "with a file to test and a destination to verify" do
|
8
5
|
setup do
|
9
|
-
FileUtils.rm_f(
|
6
|
+
FileUtils.rm_f(target_file)
|
10
7
|
end
|
11
8
|
|
12
9
|
teardown do
|
13
|
-
FileUtils.rm_f(
|
10
|
+
FileUtils.rm_f(target_file)
|
14
11
|
end
|
15
12
|
|
16
13
|
# run the worker in the foreground and the requests in the background
|
17
|
-
should "run a test" do
|
14
|
+
should "run a test in the foreground" do
|
18
15
|
num_runners = 4
|
19
16
|
pipe = Hydra::Pipe.new
|
20
17
|
child = Process.fork do
|
21
18
|
request_a_file_and_verify_completion(pipe, num_runners)
|
22
|
-
pipe.close
|
23
19
|
end
|
24
20
|
run_the_worker(pipe, num_runners)
|
25
21
|
Process.wait(child)
|
26
22
|
end
|
27
23
|
|
28
24
|
# inverse of the above test to run the worker in the background
|
29
|
-
should "
|
25
|
+
should "run a test in the background" do
|
30
26
|
num_runners = 4
|
31
27
|
pipe = Hydra::Pipe.new
|
32
28
|
child = Process.fork do
|
@@ -34,15 +30,13 @@ class WorkerTest < Test::Unit::TestCase
|
|
34
30
|
end
|
35
31
|
request_a_file_and_verify_completion(pipe, num_runners)
|
36
32
|
Process.wait(child)
|
37
|
-
pipe.close
|
38
33
|
end
|
39
34
|
end
|
40
35
|
|
41
36
|
module WorkerTestHelper
|
42
37
|
def run_the_worker(pipe, num_runners)
|
43
38
|
pipe.identify_as_child
|
44
|
-
Hydra::Worker.new(pipe, num_runners)
|
45
|
-
pipe.close
|
39
|
+
Hydra::Worker.new({:io => pipe, :runners => num_runners})
|
46
40
|
end
|
47
41
|
|
48
42
|
def request_a_file_and_verify_completion(pipe, num_runners)
|
@@ -50,15 +44,14 @@ class WorkerTest < Test::Unit::TestCase
|
|
50
44
|
num_runners.times do
|
51
45
|
assert pipe.gets.is_a?(Hydra::Messages::Worker::RequestFile)
|
52
46
|
end
|
53
|
-
pipe.write(Hydra::Messages::
|
47
|
+
pipe.write(Hydra::Messages::Master::RunFile.new(:file => test_file))
|
54
48
|
|
55
|
-
|
56
|
-
assert response.is_a?(Hydra::Messages::Worker::Results)
|
49
|
+
assert pipe.gets.is_a?(Hydra::Messages::Worker::Results)
|
57
50
|
|
58
|
-
pipe.write(Hydra::Messages::
|
51
|
+
pipe.write(Hydra::Messages::Master::Shutdown.new)
|
59
52
|
|
60
|
-
assert File.exists?(
|
61
|
-
assert_equal "HYDRA", File.read(
|
53
|
+
assert File.exists?(target_file)
|
54
|
+
assert_equal "HYDRA", File.read(target_file)
|
62
55
|
end
|
63
56
|
end
|
64
57
|
include WorkerTestHelper
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hydra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nick Gauthier
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-01-
|
12
|
+
date: 2010-01-29 00:00:00 -05:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -52,7 +52,9 @@ files:
|
|
52
52
|
- VERSION
|
53
53
|
- hydra.gemspec
|
54
54
|
- lib/hydra.rb
|
55
|
+
- lib/hydra/master.rb
|
55
56
|
- lib/hydra/message.rb
|
57
|
+
- lib/hydra/message/master_messages.rb
|
56
58
|
- lib/hydra/message/runner_messages.rb
|
57
59
|
- lib/hydra/message/worker_messages.rb
|
58
60
|
- lib/hydra/messaging_io.rb
|
@@ -63,6 +65,7 @@ files:
|
|
63
65
|
- test/fixtures/assert_true.rb
|
64
66
|
- test/fixtures/echo_the_dolphin.rb
|
65
67
|
- test/fixtures/write_file.rb
|
68
|
+
- test/master_test.rb
|
66
69
|
- test/message_test.rb
|
67
70
|
- test/pipe_test.rb
|
68
71
|
- test/runner_test.rb
|
@@ -104,6 +107,7 @@ test_files:
|
|
104
107
|
- test/fixtures/write_file.rb
|
105
108
|
- test/fixtures/assert_true.rb
|
106
109
|
- test/fixtures/echo_the_dolphin.rb
|
110
|
+
- test/master_test.rb
|
107
111
|
- test/worker_test.rb
|
108
112
|
- test/runner_test.rb
|
109
113
|
- test/pipe_test.rb
|