drbqs 0.0.15 → 0.0.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (96) hide show
  1. data/.document +3 -0
  2. data/README.md +137 -128
  3. data/VERSION +1 -1
  4. data/docs/FormatExecute.md +119 -0
  5. data/docs/GettingStarted.md +242 -0
  6. data/drbqs.gemspec +36 -13
  7. data/example/command/server_def.rb +4 -5
  8. data/example/execute/execute.rb +41 -0
  9. data/example/execute/server.rb +14 -0
  10. data/example/execute/task.rb +0 -0
  11. data/example/mandelbrot/README.md +15 -0
  12. data/example/mandelbrot/execute.rb +10 -0
  13. data/example/mandelbrot/mandelbrot.rb +56 -0
  14. data/example/mandelbrot/server.rb +49 -0
  15. data/example/server/server.rb +3 -6
  16. data/example/simple/README.md +18 -0
  17. data/example/simple/execute.rb +11 -0
  18. data/example/simple/server.rb +8 -0
  19. data/example/simple/task.rb +11 -0
  20. data/example/sum/server_def.rb +1 -1
  21. data/example/sum2/execute_def.rb +21 -8
  22. data/example/sum2/server_def.rb +8 -7
  23. data/example/transfer/file.rb +42 -8
  24. data/example/transfer/server_def.rb +43 -9
  25. data/lib/drbqs.rb +1 -1
  26. data/lib/drbqs/command_line/command_execute.rb +3 -3
  27. data/lib/drbqs/command_line/command_line.rb +1 -1
  28. data/lib/drbqs/execute/execute_node.rb +50 -0
  29. data/lib/drbqs/execute/process_define.rb +102 -54
  30. data/lib/drbqs/execute/register.rb +241 -87
  31. data/lib/drbqs/execute/server_define.rb +69 -58
  32. data/lib/drbqs/ext/task.rb +2 -0
  33. data/lib/drbqs/ext/task/command_task.rb +43 -0
  34. data/lib/drbqs/manage/manage.rb +5 -4
  35. data/lib/drbqs/manage/ssh_shell.rb +2 -8
  36. data/lib/drbqs/node/connection.rb +1 -1
  37. data/lib/drbqs/node/node.rb +8 -14
  38. data/lib/drbqs/node/task_client.rb +1 -1
  39. data/lib/drbqs/server/history.rb +5 -1
  40. data/lib/drbqs/server/message.rb +7 -34
  41. data/lib/drbqs/server/queue.rb +14 -2
  42. data/lib/drbqs/server/server.rb +86 -43
  43. data/lib/drbqs/server/server_hook.rb +3 -0
  44. data/lib/drbqs/server/test/node.rb +1 -1
  45. data/lib/drbqs/server/test/prof.rb +50 -0
  46. data/lib/drbqs/server/test/server.rb +2 -2
  47. data/lib/drbqs/server/transfer_setting.rb +23 -11
  48. data/lib/drbqs/setting/base.rb +15 -0
  49. data/lib/drbqs/setting/data_container.rb +1 -1
  50. data/lib/drbqs/setting/execute.rb +3 -3
  51. data/lib/drbqs/setting/node.rb +1 -1
  52. data/lib/drbqs/setting/server.rb +2 -2
  53. data/lib/drbqs/task/registrar.rb +39 -0
  54. data/lib/drbqs/task/task.rb +139 -59
  55. data/lib/drbqs/task/task_generator.rb +93 -116
  56. data/lib/drbqs/utility/misc.rb +15 -10
  57. data/lib/drbqs/utility/temporary.rb +7 -2
  58. data/lib/drbqs/utility/transfer/transfer.rb +81 -0
  59. data/lib/drbqs/utility/transfer/transfer_client.rb +68 -69
  60. data/lib/drbqs/utility/transfer/transfer_client_connect.rb +83 -0
  61. data/lib/drbqs/utility/transfer/transfer_file_list.rb +40 -0
  62. data/spec/execute/def/execute1.rb +4 -4
  63. data/spec/execute/def/execute2.rb +24 -0
  64. data/spec/execute/process_define_spec.rb +43 -6
  65. data/spec/execute/register_spec.rb +403 -9
  66. data/spec/execute/server_define_spec.rb +1 -1
  67. data/spec/ext/task/command_task_spec.rb +16 -0
  68. data/spec/integration_test/01_basic_usage_spec.rb +1 -1
  69. data/spec/integration_test/02_use_generator_spec.rb +2 -2
  70. data/spec/integration_test/04_use_unix_domain_spec.rb +1 -1
  71. data/spec/integration_test/05_server_exit_signal_spec.rb +1 -1
  72. data/spec/integration_test/06_node_exit_after_task_spec.rb +4 -4
  73. data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +2 -2
  74. data/spec/integration_test/09_server_process_data_spec.rb +1 -1
  75. data/spec/integration_test/definition/server01.rb +4 -5
  76. data/spec/integration_test/definition/server02.rb +2 -4
  77. data/spec/node/node_spec.rb +34 -0
  78. data/spec/server/message_spec.rb +1 -1
  79. data/spec/server/queue_spec.rb +34 -7
  80. data/spec/server/server_spec.rb +21 -9
  81. data/spec/server/transfer_setting_spec.rb +59 -24
  82. data/spec/setting/base_spec.rb +11 -0
  83. data/spec/setting/data_container_spec.rb +8 -0
  84. data/spec/spec_helper.rb +1 -7
  85. data/spec/task/registrar_spec.rb +34 -0
  86. data/spec/task/task_generator_spec.rb +15 -15
  87. data/spec/task/task_spec.rb +132 -23
  88. data/spec/utility/misc_spec.rb +2 -2
  89. data/spec/utility/transfer/transfer_client_connect_spec.rb +90 -0
  90. data/spec/utility/transfer/transfer_file_list_spec.rb +27 -0
  91. data/spec/{task/file_transfer_spec.rb → utility/transfer/transfer_spec.rb} +24 -24
  92. metadata +66 -45
  93. data/lib/drbqs/manage/execute_node.rb +0 -50
  94. data/lib/drbqs/server/prof.rb +0 -48
  95. data/lib/drbqs/task/command_task.rb +0 -43
  96. data/lib/drbqs/utility/transfer/file_transfer.rb +0 -73
@@ -1,50 +0,0 @@
1
- module DRbQS
2
-
3
- class ExecuteNode
4
- attr_reader :pid
5
-
6
- def initialize(uri, log_prefix, log_level, node_opts = {})
7
- @uri = uri
8
- @log_level = log_level
9
- if log_prefix
10
- @fname = FileName.new(log_prefix, :position => :suffix, :type => :time,
11
- :add => :always, :directory => :parent,
12
- :format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
13
- else
14
- @fname = nil
15
- end
16
- @pid = []
17
- @node_opts = node_opts
18
- end
19
-
20
- def get_log_file
21
- if @fname
22
- return @fname.create
23
- end
24
- return STDOUT
25
- end
26
- private :get_log_file
27
-
28
- def create_process
29
- @pid << fork do
30
- opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
31
- node = DRbQS::Node.new(@uri, opts)
32
- node.connect
33
- node.calculate
34
- end
35
- end
36
- private :create_process
37
-
38
- def execute(process_num, interval = 0)
39
- process_num.times do |i|
40
- create_process
41
- sleep(interval) if interval > 0
42
- end
43
- end
44
-
45
- def wait
46
- Process.waitall
47
- end
48
- end
49
-
50
- end
@@ -1,48 +0,0 @@
1
- require 'ruby-prof'
2
-
3
- module DRbQS
4
- class Prof
5
- PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
6
-
7
- # :flat
8
- # :graph
9
- # :graphhtml
10
- # :calltree
11
- def initialize(printer_type, output)
12
- @printer_type = printer_type
13
- unless PRINTER_TYPE.include?(@printer_type)
14
- raise "Invalid printer type: #{@printer_type.inspect}"
15
- end
16
- @output = output
17
- end
18
-
19
- def get_printer(result)
20
- case @printer_type
21
- when :flat
22
- RubyProf::FlatPrinter.new(result)
23
- when :graph
24
- RubyProf::GraphPrinter.new(result)
25
- when :graphhtml
26
- RubyProf::GraphHtmlPrinter.new(result)
27
- when :calltree
28
- RubyProf::CallTreePrinter.new(result)
29
- end
30
- end
31
- private :get_printer
32
-
33
- def start
34
- RubyProf.start
35
- end
36
-
37
- def finish
38
- printer = get_printer(RubyProf.stop)
39
- if IO === @output
40
- printer.print(@output)
41
- else
42
- Kernel.open(@output, 'w') do |f|
43
- printer.print(f)
44
- end
45
- end
46
- end
47
- end
48
- end
@@ -1,43 +0,0 @@
1
- module DRbQS
2
- # Execute a command and transfer files if needed.
3
- class CommandExecute
4
-
5
- # :transfer String or Array
6
- # :compress true or false
7
- def initialize(cmd, opts = {})
8
- @cmd = cmd
9
- unless (Array === @cmd || String === @cmd)
10
- raise ArgumentError, "Invalid command: #{@cmd.inspect}"
11
- end
12
- @transfer = opts[:transfer]
13
- @compress = opts[:compress]
14
- end
15
-
16
- def exec
17
- case @cmd
18
- when Array
19
- @cmd.each { |c| system(c) }
20
- when String
21
- system(@cmd)
22
- end
23
- exit_status = $?.exitstatus
24
- if @transfer
25
- if @transfer.respond_to?(:each)
26
- @transfer.each { |path| DRbQS::FileTransfer.enqueue(path, :compress => @compress) }
27
- else
28
- DRbQS::FileTransfer.enqueue(@transfer, :compress => @compress)
29
- end
30
- end
31
- exit_status
32
- end
33
- end
34
-
35
- # Class to define tasks such that we execute a command.
36
- class CommandTask < Task
37
-
38
- # &hook takes a server instance and exit number of command.
39
- def initialize(cmd, opts = {}, &hook)
40
- super(CommandExecute.new(cmd, opts), :exec, &hook)
41
- end
42
- end
43
- end
@@ -1,73 +0,0 @@
1
- module DRbQS
2
- # To compress files, we use gzip and tar command.
3
- # Note that if we compress files then we delete the source files.
4
- module FileTransfer
5
- @@files = Queue.new
6
-
7
- # If opts[:compress] is true then the file of +path+ is compressed before tranfering.
8
- def self.enqueue(path, opts = {})
9
- if opts[:rename]
10
- new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
11
- FileUtils.mv(path, new_path)
12
- path = new_path
13
- end
14
- if opts[:compress]
15
- if File.directory?(path)
16
- gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
17
- cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
18
- else
19
- gz_path = path + '.gz'
20
- cmd = "gzip --best #{path} > /dev/null 2>&1"
21
- end
22
- if File.exist?(gz_path)
23
- raise "File has already existed: #{gz_path}"
24
- elsif !system(cmd)
25
- raise "Can not compress: #{path}"
26
- end
27
- FileUtils.rm_r(path) if File.exist?(path)
28
- path_to_send = gz_path
29
- else
30
- path_to_send = path
31
- end
32
- @@files.enq(path_to_send)
33
- File.basename(path_to_send)
34
- end
35
-
36
- def self.compress_enqueue(path)
37
- self.enqueue(path, :compress => true)
38
- end
39
-
40
- def self.dequeue
41
- @@files.deq
42
- end
43
-
44
- def self.empty?
45
- @@files.empty?
46
- end
47
-
48
- def self.dequeue_all
49
- files = []
50
- until self.empty?
51
- files << self.dequeue
52
- end
53
- files.empty? ? nil : files
54
- end
55
-
56
- def self.decompress(server, filename)
57
- dir = server.transfer_directory
58
- path = File.join(dir, filename)
59
- if File.exist?(path)
60
- case path
61
- when /\.tar\.gz$/
62
- cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
63
- when /\.gz$/
64
- cmd = "gunzip #{path} > /dev/null 2>&1"
65
- else
66
- cmd = nil
67
- end
68
- system(cmd) if cmd
69
- end
70
- end
71
- end
72
-
73
- end