drbqs 0.0.15 → 0.0.16
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +3 -0
- data/README.md +137 -128
- data/VERSION +1 -1
- data/docs/FormatExecute.md +119 -0
- data/docs/GettingStarted.md +242 -0
- data/drbqs.gemspec +36 -13
- data/example/command/server_def.rb +4 -5
- data/example/execute/execute.rb +41 -0
- data/example/execute/server.rb +14 -0
- data/example/execute/task.rb +0 -0
- data/example/mandelbrot/README.md +15 -0
- data/example/mandelbrot/execute.rb +10 -0
- data/example/mandelbrot/mandelbrot.rb +56 -0
- data/example/mandelbrot/server.rb +49 -0
- data/example/server/server.rb +3 -6
- data/example/simple/README.md +18 -0
- data/example/simple/execute.rb +11 -0
- data/example/simple/server.rb +8 -0
- data/example/simple/task.rb +11 -0
- data/example/sum/server_def.rb +1 -1
- data/example/sum2/execute_def.rb +21 -8
- data/example/sum2/server_def.rb +8 -7
- data/example/transfer/file.rb +42 -8
- data/example/transfer/server_def.rb +43 -9
- data/lib/drbqs.rb +1 -1
- data/lib/drbqs/command_line/command_execute.rb +3 -3
- data/lib/drbqs/command_line/command_line.rb +1 -1
- data/lib/drbqs/execute/execute_node.rb +50 -0
- data/lib/drbqs/execute/process_define.rb +102 -54
- data/lib/drbqs/execute/register.rb +241 -87
- data/lib/drbqs/execute/server_define.rb +69 -58
- data/lib/drbqs/ext/task.rb +2 -0
- data/lib/drbqs/ext/task/command_task.rb +43 -0
- data/lib/drbqs/manage/manage.rb +5 -4
- data/lib/drbqs/manage/ssh_shell.rb +2 -8
- data/lib/drbqs/node/connection.rb +1 -1
- data/lib/drbqs/node/node.rb +8 -14
- data/lib/drbqs/node/task_client.rb +1 -1
- data/lib/drbqs/server/history.rb +5 -1
- data/lib/drbqs/server/message.rb +7 -34
- data/lib/drbqs/server/queue.rb +14 -2
- data/lib/drbqs/server/server.rb +86 -43
- data/lib/drbqs/server/server_hook.rb +3 -0
- data/lib/drbqs/server/test/node.rb +1 -1
- data/lib/drbqs/server/test/prof.rb +50 -0
- data/lib/drbqs/server/test/server.rb +2 -2
- data/lib/drbqs/server/transfer_setting.rb +23 -11
- data/lib/drbqs/setting/base.rb +15 -0
- data/lib/drbqs/setting/data_container.rb +1 -1
- data/lib/drbqs/setting/execute.rb +3 -3
- data/lib/drbqs/setting/node.rb +1 -1
- data/lib/drbqs/setting/server.rb +2 -2
- data/lib/drbqs/task/registrar.rb +39 -0
- data/lib/drbqs/task/task.rb +139 -59
- data/lib/drbqs/task/task_generator.rb +93 -116
- data/lib/drbqs/utility/misc.rb +15 -10
- data/lib/drbqs/utility/temporary.rb +7 -2
- data/lib/drbqs/utility/transfer/transfer.rb +81 -0
- data/lib/drbqs/utility/transfer/transfer_client.rb +68 -69
- data/lib/drbqs/utility/transfer/transfer_client_connect.rb +83 -0
- data/lib/drbqs/utility/transfer/transfer_file_list.rb +40 -0
- data/spec/execute/def/execute1.rb +4 -4
- data/spec/execute/def/execute2.rb +24 -0
- data/spec/execute/process_define_spec.rb +43 -6
- data/spec/execute/register_spec.rb +403 -9
- data/spec/execute/server_define_spec.rb +1 -1
- data/spec/ext/task/command_task_spec.rb +16 -0
- data/spec/integration_test/01_basic_usage_spec.rb +1 -1
- data/spec/integration_test/02_use_generator_spec.rb +2 -2
- data/spec/integration_test/04_use_unix_domain_spec.rb +1 -1
- data/spec/integration_test/05_server_exit_signal_spec.rb +1 -1
- data/spec/integration_test/06_node_exit_after_task_spec.rb +4 -4
- data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +2 -2
- data/spec/integration_test/09_server_process_data_spec.rb +1 -1
- data/spec/integration_test/definition/server01.rb +4 -5
- data/spec/integration_test/definition/server02.rb +2 -4
- data/spec/node/node_spec.rb +34 -0
- data/spec/server/message_spec.rb +1 -1
- data/spec/server/queue_spec.rb +34 -7
- data/spec/server/server_spec.rb +21 -9
- data/spec/server/transfer_setting_spec.rb +59 -24
- data/spec/setting/base_spec.rb +11 -0
- data/spec/setting/data_container_spec.rb +8 -0
- data/spec/spec_helper.rb +1 -7
- data/spec/task/registrar_spec.rb +34 -0
- data/spec/task/task_generator_spec.rb +15 -15
- data/spec/task/task_spec.rb +132 -23
- data/spec/utility/misc_spec.rb +2 -2
- data/spec/utility/transfer/transfer_client_connect_spec.rb +90 -0
- data/spec/utility/transfer/transfer_file_list_spec.rb +27 -0
- data/spec/{task/file_transfer_spec.rb → utility/transfer/transfer_spec.rb} +24 -24
- metadata +66 -45
- data/lib/drbqs/manage/execute_node.rb +0 -50
- data/lib/drbqs/server/prof.rb +0 -48
- data/lib/drbqs/task/command_task.rb +0 -43
- data/lib/drbqs/utility/transfer/file_transfer.rb +0 -73
@@ -1,50 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
|
3
|
-
class ExecuteNode
|
4
|
-
attr_reader :pid
|
5
|
-
|
6
|
-
def initialize(uri, log_prefix, log_level, node_opts = {})
|
7
|
-
@uri = uri
|
8
|
-
@log_level = log_level
|
9
|
-
if log_prefix
|
10
|
-
@fname = FileName.new(log_prefix, :position => :suffix, :type => :time,
|
11
|
-
:add => :always, :directory => :parent,
|
12
|
-
:format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
|
13
|
-
else
|
14
|
-
@fname = nil
|
15
|
-
end
|
16
|
-
@pid = []
|
17
|
-
@node_opts = node_opts
|
18
|
-
end
|
19
|
-
|
20
|
-
def get_log_file
|
21
|
-
if @fname
|
22
|
-
return @fname.create
|
23
|
-
end
|
24
|
-
return STDOUT
|
25
|
-
end
|
26
|
-
private :get_log_file
|
27
|
-
|
28
|
-
def create_process
|
29
|
-
@pid << fork do
|
30
|
-
opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
|
31
|
-
node = DRbQS::Node.new(@uri, opts)
|
32
|
-
node.connect
|
33
|
-
node.calculate
|
34
|
-
end
|
35
|
-
end
|
36
|
-
private :create_process
|
37
|
-
|
38
|
-
def execute(process_num, interval = 0)
|
39
|
-
process_num.times do |i|
|
40
|
-
create_process
|
41
|
-
sleep(interval) if interval > 0
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def wait
|
46
|
-
Process.waitall
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
end
|
data/lib/drbqs/server/prof.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'ruby-prof'
|
2
|
-
|
3
|
-
module DRbQS
|
4
|
-
class Prof
|
5
|
-
PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
|
6
|
-
|
7
|
-
# :flat
|
8
|
-
# :graph
|
9
|
-
# :graphhtml
|
10
|
-
# :calltree
|
11
|
-
def initialize(printer_type, output)
|
12
|
-
@printer_type = printer_type
|
13
|
-
unless PRINTER_TYPE.include?(@printer_type)
|
14
|
-
raise "Invalid printer type: #{@printer_type.inspect}"
|
15
|
-
end
|
16
|
-
@output = output
|
17
|
-
end
|
18
|
-
|
19
|
-
def get_printer(result)
|
20
|
-
case @printer_type
|
21
|
-
when :flat
|
22
|
-
RubyProf::FlatPrinter.new(result)
|
23
|
-
when :graph
|
24
|
-
RubyProf::GraphPrinter.new(result)
|
25
|
-
when :graphhtml
|
26
|
-
RubyProf::GraphHtmlPrinter.new(result)
|
27
|
-
when :calltree
|
28
|
-
RubyProf::CallTreePrinter.new(result)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
private :get_printer
|
32
|
-
|
33
|
-
def start
|
34
|
-
RubyProf.start
|
35
|
-
end
|
36
|
-
|
37
|
-
def finish
|
38
|
-
printer = get_printer(RubyProf.stop)
|
39
|
-
if IO === @output
|
40
|
-
printer.print(@output)
|
41
|
-
else
|
42
|
-
Kernel.open(@output, 'w') do |f|
|
43
|
-
printer.print(f)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
# Execute a command and transfer files if needed.
|
3
|
-
class CommandExecute
|
4
|
-
|
5
|
-
# :transfer String or Array
|
6
|
-
# :compress true or false
|
7
|
-
def initialize(cmd, opts = {})
|
8
|
-
@cmd = cmd
|
9
|
-
unless (Array === @cmd || String === @cmd)
|
10
|
-
raise ArgumentError, "Invalid command: #{@cmd.inspect}"
|
11
|
-
end
|
12
|
-
@transfer = opts[:transfer]
|
13
|
-
@compress = opts[:compress]
|
14
|
-
end
|
15
|
-
|
16
|
-
def exec
|
17
|
-
case @cmd
|
18
|
-
when Array
|
19
|
-
@cmd.each { |c| system(c) }
|
20
|
-
when String
|
21
|
-
system(@cmd)
|
22
|
-
end
|
23
|
-
exit_status = $?.exitstatus
|
24
|
-
if @transfer
|
25
|
-
if @transfer.respond_to?(:each)
|
26
|
-
@transfer.each { |path| DRbQS::FileTransfer.enqueue(path, :compress => @compress) }
|
27
|
-
else
|
28
|
-
DRbQS::FileTransfer.enqueue(@transfer, :compress => @compress)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
exit_status
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
# Class to define tasks such that we execute a command.
|
36
|
-
class CommandTask < Task
|
37
|
-
|
38
|
-
# &hook takes a server instance and exit number of command.
|
39
|
-
def initialize(cmd, opts = {}, &hook)
|
40
|
-
super(CommandExecute.new(cmd, opts), :exec, &hook)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,73 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
# To compress files, we use gzip and tar command.
|
3
|
-
# Note that if we compress files then we delete the source files.
|
4
|
-
module FileTransfer
|
5
|
-
@@files = Queue.new
|
6
|
-
|
7
|
-
# If opts[:compress] is true then the file of +path+ is compressed before tranfering.
|
8
|
-
def self.enqueue(path, opts = {})
|
9
|
-
if opts[:rename]
|
10
|
-
new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
|
11
|
-
FileUtils.mv(path, new_path)
|
12
|
-
path = new_path
|
13
|
-
end
|
14
|
-
if opts[:compress]
|
15
|
-
if File.directory?(path)
|
16
|
-
gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
|
17
|
-
cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
|
18
|
-
else
|
19
|
-
gz_path = path + '.gz'
|
20
|
-
cmd = "gzip --best #{path} > /dev/null 2>&1"
|
21
|
-
end
|
22
|
-
if File.exist?(gz_path)
|
23
|
-
raise "File has already existed: #{gz_path}"
|
24
|
-
elsif !system(cmd)
|
25
|
-
raise "Can not compress: #{path}"
|
26
|
-
end
|
27
|
-
FileUtils.rm_r(path) if File.exist?(path)
|
28
|
-
path_to_send = gz_path
|
29
|
-
else
|
30
|
-
path_to_send = path
|
31
|
-
end
|
32
|
-
@@files.enq(path_to_send)
|
33
|
-
File.basename(path_to_send)
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.compress_enqueue(path)
|
37
|
-
self.enqueue(path, :compress => true)
|
38
|
-
end
|
39
|
-
|
40
|
-
def self.dequeue
|
41
|
-
@@files.deq
|
42
|
-
end
|
43
|
-
|
44
|
-
def self.empty?
|
45
|
-
@@files.empty?
|
46
|
-
end
|
47
|
-
|
48
|
-
def self.dequeue_all
|
49
|
-
files = []
|
50
|
-
until self.empty?
|
51
|
-
files << self.dequeue
|
52
|
-
end
|
53
|
-
files.empty? ? nil : files
|
54
|
-
end
|
55
|
-
|
56
|
-
def self.decompress(server, filename)
|
57
|
-
dir = server.transfer_directory
|
58
|
-
path = File.join(dir, filename)
|
59
|
-
if File.exist?(path)
|
60
|
-
case path
|
61
|
-
when /\.tar\.gz$/
|
62
|
-
cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
|
63
|
-
when /\.gz$/
|
64
|
-
cmd = "gunzip #{path} > /dev/null 2>&1"
|
65
|
-
else
|
66
|
-
cmd = nil
|
67
|
-
end
|
68
|
-
system(cmd) if cmd
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
end
|