drbqs 0.0.15 → 0.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. data/.document +3 -0
  2. data/README.md +137 -128
  3. data/VERSION +1 -1
  4. data/docs/FormatExecute.md +119 -0
  5. data/docs/GettingStarted.md +242 -0
  6. data/drbqs.gemspec +36 -13
  7. data/example/command/server_def.rb +4 -5
  8. data/example/execute/execute.rb +41 -0
  9. data/example/execute/server.rb +14 -0
  10. data/example/execute/task.rb +0 -0
  11. data/example/mandelbrot/README.md +15 -0
  12. data/example/mandelbrot/execute.rb +10 -0
  13. data/example/mandelbrot/mandelbrot.rb +56 -0
  14. data/example/mandelbrot/server.rb +49 -0
  15. data/example/server/server.rb +3 -6
  16. data/example/simple/README.md +18 -0
  17. data/example/simple/execute.rb +11 -0
  18. data/example/simple/server.rb +8 -0
  19. data/example/simple/task.rb +11 -0
  20. data/example/sum/server_def.rb +1 -1
  21. data/example/sum2/execute_def.rb +21 -8
  22. data/example/sum2/server_def.rb +8 -7
  23. data/example/transfer/file.rb +42 -8
  24. data/example/transfer/server_def.rb +43 -9
  25. data/lib/drbqs.rb +1 -1
  26. data/lib/drbqs/command_line/command_execute.rb +3 -3
  27. data/lib/drbqs/command_line/command_line.rb +1 -1
  28. data/lib/drbqs/execute/execute_node.rb +50 -0
  29. data/lib/drbqs/execute/process_define.rb +102 -54
  30. data/lib/drbqs/execute/register.rb +241 -87
  31. data/lib/drbqs/execute/server_define.rb +69 -58
  32. data/lib/drbqs/ext/task.rb +2 -0
  33. data/lib/drbqs/ext/task/command_task.rb +43 -0
  34. data/lib/drbqs/manage/manage.rb +5 -4
  35. data/lib/drbqs/manage/ssh_shell.rb +2 -8
  36. data/lib/drbqs/node/connection.rb +1 -1
  37. data/lib/drbqs/node/node.rb +8 -14
  38. data/lib/drbqs/node/task_client.rb +1 -1
  39. data/lib/drbqs/server/history.rb +5 -1
  40. data/lib/drbqs/server/message.rb +7 -34
  41. data/lib/drbqs/server/queue.rb +14 -2
  42. data/lib/drbqs/server/server.rb +86 -43
  43. data/lib/drbqs/server/server_hook.rb +3 -0
  44. data/lib/drbqs/server/test/node.rb +1 -1
  45. data/lib/drbqs/server/test/prof.rb +50 -0
  46. data/lib/drbqs/server/test/server.rb +2 -2
  47. data/lib/drbqs/server/transfer_setting.rb +23 -11
  48. data/lib/drbqs/setting/base.rb +15 -0
  49. data/lib/drbqs/setting/data_container.rb +1 -1
  50. data/lib/drbqs/setting/execute.rb +3 -3
  51. data/lib/drbqs/setting/node.rb +1 -1
  52. data/lib/drbqs/setting/server.rb +2 -2
  53. data/lib/drbqs/task/registrar.rb +39 -0
  54. data/lib/drbqs/task/task.rb +139 -59
  55. data/lib/drbqs/task/task_generator.rb +93 -116
  56. data/lib/drbqs/utility/misc.rb +15 -10
  57. data/lib/drbqs/utility/temporary.rb +7 -2
  58. data/lib/drbqs/utility/transfer/transfer.rb +81 -0
  59. data/lib/drbqs/utility/transfer/transfer_client.rb +68 -69
  60. data/lib/drbqs/utility/transfer/transfer_client_connect.rb +83 -0
  61. data/lib/drbqs/utility/transfer/transfer_file_list.rb +40 -0
  62. data/spec/execute/def/execute1.rb +4 -4
  63. data/spec/execute/def/execute2.rb +24 -0
  64. data/spec/execute/process_define_spec.rb +43 -6
  65. data/spec/execute/register_spec.rb +403 -9
  66. data/spec/execute/server_define_spec.rb +1 -1
  67. data/spec/ext/task/command_task_spec.rb +16 -0
  68. data/spec/integration_test/01_basic_usage_spec.rb +1 -1
  69. data/spec/integration_test/02_use_generator_spec.rb +2 -2
  70. data/spec/integration_test/04_use_unix_domain_spec.rb +1 -1
  71. data/spec/integration_test/05_server_exit_signal_spec.rb +1 -1
  72. data/spec/integration_test/06_node_exit_after_task_spec.rb +4 -4
  73. data/spec/integration_test/08_shutdown_unused_nodes_spec.rb +2 -2
  74. data/spec/integration_test/09_server_process_data_spec.rb +1 -1
  75. data/spec/integration_test/definition/server01.rb +4 -5
  76. data/spec/integration_test/definition/server02.rb +2 -4
  77. data/spec/node/node_spec.rb +34 -0
  78. data/spec/server/message_spec.rb +1 -1
  79. data/spec/server/queue_spec.rb +34 -7
  80. data/spec/server/server_spec.rb +21 -9
  81. data/spec/server/transfer_setting_spec.rb +59 -24
  82. data/spec/setting/base_spec.rb +11 -0
  83. data/spec/setting/data_container_spec.rb +8 -0
  84. data/spec/spec_helper.rb +1 -7
  85. data/spec/task/registrar_spec.rb +34 -0
  86. data/spec/task/task_generator_spec.rb +15 -15
  87. data/spec/task/task_spec.rb +132 -23
  88. data/spec/utility/misc_spec.rb +2 -2
  89. data/spec/utility/transfer/transfer_client_connect_spec.rb +90 -0
  90. data/spec/utility/transfer/transfer_file_list_spec.rb +27 -0
  91. data/spec/{task/file_transfer_spec.rb → utility/transfer/transfer_spec.rb} +24 -24
  92. metadata +66 -45
  93. data/lib/drbqs/manage/execute_node.rb +0 -50
  94. data/lib/drbqs/server/prof.rb +0 -48
  95. data/lib/drbqs/task/command_task.rb +0 -43
  96. data/lib/drbqs/utility/transfer/file_transfer.rb +0 -73
@@ -1,50 +0,0 @@
1
- module DRbQS
2
-
3
- class ExecuteNode
4
- attr_reader :pid
5
-
6
- def initialize(uri, log_prefix, log_level, node_opts = {})
7
- @uri = uri
8
- @log_level = log_level
9
- if log_prefix
10
- @fname = FileName.new(log_prefix, :position => :suffix, :type => :time,
11
- :add => :always, :directory => :parent,
12
- :format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
13
- else
14
- @fname = nil
15
- end
16
- @pid = []
17
- @node_opts = node_opts
18
- end
19
-
20
- def get_log_file
21
- if @fname
22
- return @fname.create
23
- end
24
- return STDOUT
25
- end
26
- private :get_log_file
27
-
28
- def create_process
29
- @pid << fork do
30
- opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
31
- node = DRbQS::Node.new(@uri, opts)
32
- node.connect
33
- node.calculate
34
- end
35
- end
36
- private :create_process
37
-
38
- def execute(process_num, interval = 0)
39
- process_num.times do |i|
40
- create_process
41
- sleep(interval) if interval > 0
42
- end
43
- end
44
-
45
- def wait
46
- Process.waitall
47
- end
48
- end
49
-
50
- end
@@ -1,48 +0,0 @@
1
- require 'ruby-prof'
2
-
3
- module DRbQS
4
- class Prof
5
- PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
6
-
7
- # :flat
8
- # :graph
9
- # :graphhtml
10
- # :calltree
11
- def initialize(printer_type, output)
12
- @printer_type = printer_type
13
- unless PRINTER_TYPE.include?(@printer_type)
14
- raise "Invalid printer type: #{@printer_type.inspect}"
15
- end
16
- @output = output
17
- end
18
-
19
- def get_printer(result)
20
- case @printer_type
21
- when :flat
22
- RubyProf::FlatPrinter.new(result)
23
- when :graph
24
- RubyProf::GraphPrinter.new(result)
25
- when :graphhtml
26
- RubyProf::GraphHtmlPrinter.new(result)
27
- when :calltree
28
- RubyProf::CallTreePrinter.new(result)
29
- end
30
- end
31
- private :get_printer
32
-
33
- def start
34
- RubyProf.start
35
- end
36
-
37
- def finish
38
- printer = get_printer(RubyProf.stop)
39
- if IO === @output
40
- printer.print(@output)
41
- else
42
- Kernel.open(@output, 'w') do |f|
43
- printer.print(f)
44
- end
45
- end
46
- end
47
- end
48
- end
@@ -1,43 +0,0 @@
1
- module DRbQS
2
- # Execute a command and transfer files if needed.
3
- class CommandExecute
4
-
5
- # :transfer String or Array
6
- # :compress true or false
7
- def initialize(cmd, opts = {})
8
- @cmd = cmd
9
- unless (Array === @cmd || String === @cmd)
10
- raise ArgumentError, "Invalid command: #{@cmd.inspect}"
11
- end
12
- @transfer = opts[:transfer]
13
- @compress = opts[:compress]
14
- end
15
-
16
- def exec
17
- case @cmd
18
- when Array
19
- @cmd.each { |c| system(c) }
20
- when String
21
- system(@cmd)
22
- end
23
- exit_status = $?.exitstatus
24
- if @transfer
25
- if @transfer.respond_to?(:each)
26
- @transfer.each { |path| DRbQS::FileTransfer.enqueue(path, :compress => @compress) }
27
- else
28
- DRbQS::FileTransfer.enqueue(@transfer, :compress => @compress)
29
- end
30
- end
31
- exit_status
32
- end
33
- end
34
-
35
- # Class to define tasks such that we execute a command.
36
- class CommandTask < Task
37
-
38
- # &hook takes a server instance and exit number of command.
39
- def initialize(cmd, opts = {}, &hook)
40
- super(CommandExecute.new(cmd, opts), :exec, &hook)
41
- end
42
- end
43
- end
@@ -1,73 +0,0 @@
1
- module DRbQS
2
- # To compress files, we use gzip and tar command.
3
- # Note that if we compress files then we delete the source files.
4
- module FileTransfer
5
- @@files = Queue.new
6
-
7
- # If opts[:compress] is true then the file of +path+ is compressed before tranfering.
8
- def self.enqueue(path, opts = {})
9
- if opts[:rename]
10
- new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
11
- FileUtils.mv(path, new_path)
12
- path = new_path
13
- end
14
- if opts[:compress]
15
- if File.directory?(path)
16
- gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
17
- cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
18
- else
19
- gz_path = path + '.gz'
20
- cmd = "gzip --best #{path} > /dev/null 2>&1"
21
- end
22
- if File.exist?(gz_path)
23
- raise "File has already existed: #{gz_path}"
24
- elsif !system(cmd)
25
- raise "Can not compress: #{path}"
26
- end
27
- FileUtils.rm_r(path) if File.exist?(path)
28
- path_to_send = gz_path
29
- else
30
- path_to_send = path
31
- end
32
- @@files.enq(path_to_send)
33
- File.basename(path_to_send)
34
- end
35
-
36
- def self.compress_enqueue(path)
37
- self.enqueue(path, :compress => true)
38
- end
39
-
40
- def self.dequeue
41
- @@files.deq
42
- end
43
-
44
- def self.empty?
45
- @@files.empty?
46
- end
47
-
48
- def self.dequeue_all
49
- files = []
50
- until self.empty?
51
- files << self.dequeue
52
- end
53
- files.empty? ? nil : files
54
- end
55
-
56
- def self.decompress(server, filename)
57
- dir = server.transfer_directory
58
- path = File.join(dir, filename)
59
- if File.exist?(path)
60
- case path
61
- when /\.tar\.gz$/
62
- cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
63
- when /\.gz$/
64
- cmd = "gunzip #{path} > /dev/null 2>&1"
65
- else
66
- cmd = nil
67
- end
68
- system(cmd) if cmd
69
- end
70
- end
71
- end
72
-
73
- end