drbqs 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +3 -0
- data/README.md +137 -128
- data/VERSION +1 -1
- data/docs/FormatExecute.md +119 -0
- data/docs/GettingStarted.md +242 -0
- data/drbqs.gemspec +36 -13
- data/example/command/server_def.rb +4 -5
- data/example/execute/execute.rb +41 -0
- data/example/execute/server.rb +14 -0
- data/example/execute/task.rb +0 -0
- data/example/mandelbrot/README.md +15 -0
- data/example/mandelbrot/execute.rb +10 -0
- data/example/mandelbrot/mandelbrot.rb +56 -0
- data/example/mandelbrot/server.rb +49 -0
- data/example/server/server.rb +3 -6
- data/example/simple/README.md +18 -0
- data/example/simple/execute.rb +11 -0
- data/example/simple/server.rb +8 -0
- data/example/simple/task.rb +11 -0
- data/example/sum/server_def.rb +1 -1
- data/example/sum2/execute_def.rb +21 -8
- data/example/sum2/server_def.rb +8 -7
- data/example/transfer/file.rb +42 -8
- data/example/transfer/server_def.rb +43 -9
- data/lib/drbqs.rb +1 -1
- data/lib/drbqs/command_line/command_execute.rb +3 -3
- data/lib/drbqs/command_line/command_line.rb +1 -1
- data/lib/drbqs/execute/execute_node.rb +50 -0
- data/lib/drbqs/execute/process_define.rb +102 -54
- data/lib/drbqs/execute/register.rb +241 -87
- data/lib/drbqs/execute/server_define.rb +69 -58
- data/lib/drbqs/ext/task.rb +2 -0
- data/lib/drbqs/ext/task/command_task.rb +43 -0
- data/lib/drbqs/manage/manage.rb +5 -4
- data/lib/drbqs/manage/ssh_shell.rb +2 -8
- data/lib/drbqs/node/connection.rb +1 -1
- data/lib/drbqs/node/node.rb +8 -14
- data/lib/drbqs/node/task_client.rb +1 -1
- data/lib/drbqs/server/history.rb +5 -1
- data/lib/drbqs/server/message.rb +7 -34
- data/lib/drbqs/server/queue.rb +14 -2
- data/lib/drbqs/server/server.rb +86 -43
- data/lib/drbqs/server/server_hook.rb +3 -0
- data/lib/drbqs/server/test/node.rb +1 -1
- data/lib/drbqs/server/test/prof.rb +50 -0
- data/lib/drbqs/server/test/server.rb +2 -2
- data/lib/drbqs/server/transfer_setting.rb +23 -11
- data/lib/drbqs/setting/base.rb +15 -0
- data/lib/drbqs/setting/data_container.rb +1 -1
- data/lib/drbqs/setting/execute.rb +3 -3
- data/lib/drbqs/setting/node.rb +1 -1
- data/lib/drbqs/setting/server.rb +2 -2
- data/lib/drbqs/task/registrar.rb +39 -0
- data/lib/drbqs/task/task.rb +139 -59
- data/lib/drbqs/task/task_generator.rb +93 -116
- data/lib/drbqs/utility/misc.rb +15 -10
- data/lib/drbqs/utility/temporary.rb +7 -2
- data/lib/drbqs/utility/transfer/transfer.rb +81 -0
- data/lib/drbqs/utility/transfer/transfer_client.rb +68 -69
- data/lib/drbqs/utility/transfer/transfer_client_connect.rb +83 -0
- data/lib/drbqs/utility/transfer/transfer_file_list.rb +40 -0
- data/spec/execute/def/execute1.rb +4 -4
- data/spec/execute/def/execute2.rb +24 -0
- data/spec/execute/process_define_spec.rb +43 -6
- data/spec/execute/register_spec.rb +403 -9
- data/spec/execute/server_define_spec.rb +1 -1
- data/spec/ext/task/command_task_spec.rb +16 -0
- data/spec/integration_test/01_basic_usage_spec.rb +1 -1
- data/spec/integration_test/02_use_generator_spec.rb +2 -2
- data/spec/integration_test/04_use_unix_domain_spec.rb +1 -1
- data/spec/integration_test/05_server_exit_signal_spec.rb +1 -1
- data/spec/integration_test/06_node_exit_after_task_spec.rb +4 -4
- data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +2 -2
- data/spec/integration_test/09_server_process_data_spec.rb +1 -1
- data/spec/integration_test/definition/server01.rb +4 -5
- data/spec/integration_test/definition/server02.rb +2 -4
- data/spec/node/node_spec.rb +34 -0
- data/spec/server/message_spec.rb +1 -1
- data/spec/server/queue_spec.rb +34 -7
- data/spec/server/server_spec.rb +21 -9
- data/spec/server/transfer_setting_spec.rb +59 -24
- data/spec/setting/base_spec.rb +11 -0
- data/spec/setting/data_container_spec.rb +8 -0
- data/spec/spec_helper.rb +1 -7
- data/spec/task/registrar_spec.rb +34 -0
- data/spec/task/task_generator_spec.rb +15 -15
- data/spec/task/task_spec.rb +132 -23
- data/spec/utility/misc_spec.rb +2 -2
- data/spec/utility/transfer/transfer_client_connect_spec.rb +90 -0
- data/spec/utility/transfer/transfer_file_list_spec.rb +27 -0
- data/spec/{task/file_transfer_spec.rb → utility/transfer/transfer_spec.rb} +24 -24
- metadata +66 -45
- data/lib/drbqs/manage/execute_node.rb +0 -50
- data/lib/drbqs/server/prof.rb +0 -48
- data/lib/drbqs/task/command_task.rb +0 -43
- data/lib/drbqs/utility/transfer/file_transfer.rb +0 -73
@@ -1,50 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
|
3
|
-
class ExecuteNode
|
4
|
-
attr_reader :pid
|
5
|
-
|
6
|
-
def initialize(uri, log_prefix, log_level, node_opts = {})
|
7
|
-
@uri = uri
|
8
|
-
@log_level = log_level
|
9
|
-
if log_prefix
|
10
|
-
@fname = FileName.new(log_prefix, :position => :suffix, :type => :time,
|
11
|
-
:add => :always, :directory => :parent,
|
12
|
-
:format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
|
13
|
-
else
|
14
|
-
@fname = nil
|
15
|
-
end
|
16
|
-
@pid = []
|
17
|
-
@node_opts = node_opts
|
18
|
-
end
|
19
|
-
|
20
|
-
def get_log_file
|
21
|
-
if @fname
|
22
|
-
return @fname.create
|
23
|
-
end
|
24
|
-
return STDOUT
|
25
|
-
end
|
26
|
-
private :get_log_file
|
27
|
-
|
28
|
-
def create_process
|
29
|
-
@pid << fork do
|
30
|
-
opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
|
31
|
-
node = DRbQS::Node.new(@uri, opts)
|
32
|
-
node.connect
|
33
|
-
node.calculate
|
34
|
-
end
|
35
|
-
end
|
36
|
-
private :create_process
|
37
|
-
|
38
|
-
def execute(process_num, interval = 0)
|
39
|
-
process_num.times do |i|
|
40
|
-
create_process
|
41
|
-
sleep(interval) if interval > 0
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def wait
|
46
|
-
Process.waitall
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
end
|
data/lib/drbqs/server/prof.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'ruby-prof'
|
2
|
-
|
3
|
-
module DRbQS
|
4
|
-
class Prof
|
5
|
-
PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
|
6
|
-
|
7
|
-
# :flat
|
8
|
-
# :graph
|
9
|
-
# :graphhtml
|
10
|
-
# :calltree
|
11
|
-
def initialize(printer_type, output)
|
12
|
-
@printer_type = printer_type
|
13
|
-
unless PRINTER_TYPE.include?(@printer_type)
|
14
|
-
raise "Invalid printer type: #{@printer_type.inspect}"
|
15
|
-
end
|
16
|
-
@output = output
|
17
|
-
end
|
18
|
-
|
19
|
-
def get_printer(result)
|
20
|
-
case @printer_type
|
21
|
-
when :flat
|
22
|
-
RubyProf::FlatPrinter.new(result)
|
23
|
-
when :graph
|
24
|
-
RubyProf::GraphPrinter.new(result)
|
25
|
-
when :graphhtml
|
26
|
-
RubyProf::GraphHtmlPrinter.new(result)
|
27
|
-
when :calltree
|
28
|
-
RubyProf::CallTreePrinter.new(result)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
private :get_printer
|
32
|
-
|
33
|
-
def start
|
34
|
-
RubyProf.start
|
35
|
-
end
|
36
|
-
|
37
|
-
def finish
|
38
|
-
printer = get_printer(RubyProf.stop)
|
39
|
-
if IO === @output
|
40
|
-
printer.print(@output)
|
41
|
-
else
|
42
|
-
Kernel.open(@output, 'w') do |f|
|
43
|
-
printer.print(f)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
# Execute a command and transfer files if needed.
|
3
|
-
class CommandExecute
|
4
|
-
|
5
|
-
# :transfer String or Array
|
6
|
-
# :compress true or false
|
7
|
-
def initialize(cmd, opts = {})
|
8
|
-
@cmd = cmd
|
9
|
-
unless (Array === @cmd || String === @cmd)
|
10
|
-
raise ArgumentError, "Invalid command: #{@cmd.inspect}"
|
11
|
-
end
|
12
|
-
@transfer = opts[:transfer]
|
13
|
-
@compress = opts[:compress]
|
14
|
-
end
|
15
|
-
|
16
|
-
def exec
|
17
|
-
case @cmd
|
18
|
-
when Array
|
19
|
-
@cmd.each { |c| system(c) }
|
20
|
-
when String
|
21
|
-
system(@cmd)
|
22
|
-
end
|
23
|
-
exit_status = $?.exitstatus
|
24
|
-
if @transfer
|
25
|
-
if @transfer.respond_to?(:each)
|
26
|
-
@transfer.each { |path| DRbQS::FileTransfer.enqueue(path, :compress => @compress) }
|
27
|
-
else
|
28
|
-
DRbQS::FileTransfer.enqueue(@transfer, :compress => @compress)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
exit_status
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
# Class to define tasks such that we execute a command.
|
36
|
-
class CommandTask < Task
|
37
|
-
|
38
|
-
# &hook takes a server instance and exit number of command.
|
39
|
-
def initialize(cmd, opts = {}, &hook)
|
40
|
-
super(CommandExecute.new(cmd, opts), :exec, &hook)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,73 +0,0 @@
|
|
1
|
-
module DRbQS
|
2
|
-
# To compress files, we use gzip and tar command.
|
3
|
-
# Note that if we compress files then we delete the source files.
|
4
|
-
module FileTransfer
|
5
|
-
@@files = Queue.new
|
6
|
-
|
7
|
-
# If opts[:compress] is true then the file of +path+ is compressed before tranfering.
|
8
|
-
def self.enqueue(path, opts = {})
|
9
|
-
if opts[:rename]
|
10
|
-
new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
|
11
|
-
FileUtils.mv(path, new_path)
|
12
|
-
path = new_path
|
13
|
-
end
|
14
|
-
if opts[:compress]
|
15
|
-
if File.directory?(path)
|
16
|
-
gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
|
17
|
-
cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
|
18
|
-
else
|
19
|
-
gz_path = path + '.gz'
|
20
|
-
cmd = "gzip --best #{path} > /dev/null 2>&1"
|
21
|
-
end
|
22
|
-
if File.exist?(gz_path)
|
23
|
-
raise "File has already existed: #{gz_path}"
|
24
|
-
elsif !system(cmd)
|
25
|
-
raise "Can not compress: #{path}"
|
26
|
-
end
|
27
|
-
FileUtils.rm_r(path) if File.exist?(path)
|
28
|
-
path_to_send = gz_path
|
29
|
-
else
|
30
|
-
path_to_send = path
|
31
|
-
end
|
32
|
-
@@files.enq(path_to_send)
|
33
|
-
File.basename(path_to_send)
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.compress_enqueue(path)
|
37
|
-
self.enqueue(path, :compress => true)
|
38
|
-
end
|
39
|
-
|
40
|
-
def self.dequeue
|
41
|
-
@@files.deq
|
42
|
-
end
|
43
|
-
|
44
|
-
def self.empty?
|
45
|
-
@@files.empty?
|
46
|
-
end
|
47
|
-
|
48
|
-
def self.dequeue_all
|
49
|
-
files = []
|
50
|
-
until self.empty?
|
51
|
-
files << self.dequeue
|
52
|
-
end
|
53
|
-
files.empty? ? nil : files
|
54
|
-
end
|
55
|
-
|
56
|
-
def self.decompress(server, filename)
|
57
|
-
dir = server.transfer_directory
|
58
|
-
path = File.join(dir, filename)
|
59
|
-
if File.exist?(path)
|
60
|
-
case path
|
61
|
-
when /\.tar\.gz$/
|
62
|
-
cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
|
63
|
-
when /\.gz$/
|
64
|
-
cmd = "gunzip #{path} > /dev/null 2>&1"
|
65
|
-
else
|
66
|
-
cmd = nil
|
67
|
-
end
|
68
|
-
system(cmd) if cmd
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
end
|