drbqs 0.0.13 → 0.0.14

Sign up to get free protection for your applications and to get access to all the features.
Files changed (115) hide show
  1. data/Gemfile +10 -7
  2. data/README.md +52 -11
  3. data/Rakefile +32 -10
  4. data/VERSION +1 -1
  5. data/bin/drbqs-manage +3 -93
  6. data/bin/drbqs-node +3 -89
  7. data/bin/drbqs-server +3 -117
  8. data/bin/drbqs-ssh +6 -0
  9. data/drbqs.gemspec +118 -97
  10. data/example/README.md +2 -2
  11. data/lib/drbqs/config/config.rb +88 -0
  12. data/lib/drbqs/config/process_list.rb +194 -0
  13. data/lib/drbqs/config/ssh_host.rb +41 -0
  14. data/lib/drbqs/{execute_node.rb → manage/execute_node.rb} +6 -4
  15. data/lib/drbqs/manage/manage.rb +100 -0
  16. data/lib/drbqs/manage/send_signal.rb +45 -0
  17. data/lib/drbqs/manage/ssh_execute.rb +23 -0
  18. data/lib/drbqs/manage/ssh_shell.rb +143 -0
  19. data/lib/drbqs/node/connection.rb +69 -0
  20. data/lib/drbqs/{client.rb → node/node.rb} +48 -18
  21. data/lib/drbqs/node/task_client.rb +94 -0
  22. data/lib/drbqs/server/acl_file.rb +15 -0
  23. data/lib/drbqs/server/check_alive.rb +23 -0
  24. data/lib/drbqs/server/history.rb +49 -0
  25. data/lib/drbqs/server/message.rb +142 -0
  26. data/lib/drbqs/server/node_list.rb +59 -0
  27. data/lib/drbqs/server/queue.rb +128 -0
  28. data/lib/drbqs/{server.rb → server/server.rb} +66 -74
  29. data/lib/drbqs/server/server_hook.rb +72 -0
  30. data/lib/drbqs/server/transfer_setting.rb +30 -0
  31. data/lib/drbqs/task/command_task.rb +43 -0
  32. data/lib/drbqs/{task.rb → task/task.rb} +18 -39
  33. data/lib/drbqs/{task_generator.rb → task/task_generator.rb} +2 -0
  34. data/lib/drbqs/utility/argument.rb +27 -0
  35. data/lib/drbqs/utility/command_line/command_base.rb +27 -0
  36. data/lib/drbqs/utility/command_line/command_manage.rb +121 -0
  37. data/lib/drbqs/utility/command_line/command_node.rb +103 -0
  38. data/lib/drbqs/utility/command_line/command_server.rb +165 -0
  39. data/lib/drbqs/utility/command_line/command_ssh.rb +126 -0
  40. data/lib/drbqs/utility/command_line.rb +15 -0
  41. data/lib/drbqs/utility/misc.rb +72 -0
  42. data/lib/drbqs/{server_define.rb → utility/server_define.rb} +23 -8
  43. data/lib/drbqs/utility/temporary.rb +49 -0
  44. data/lib/drbqs/{ssh/transfer.rb → utility/transfer/file_transfer.rb} +18 -58
  45. data/lib/drbqs/utility/transfer/transfer_client.rb +90 -0
  46. data/lib/drbqs.rb +10 -22
  47. data/spec/config/config_spec.rb +84 -0
  48. data/spec/config/process_list_spec.rb +149 -0
  49. data/spec/config/ssh_host_spec.rb +81 -0
  50. data/spec/integration_test/01_basic_usage_spec.rb +54 -0
  51. data/spec/integration_test/02_use_generator_spec.rb +53 -0
  52. data/spec/integration_test/03_use_temporary_file_spec.rb +26 -0
  53. data/spec/integration_test/04_use_unix_domain_spec.rb +34 -0
  54. data/spec/integration_test/05_server_exit_signal_spec.rb +23 -0
  55. data/spec/integration_test/06_node_exit_after_task_spec.rb +42 -0
  56. data/spec/integration_test/07_command_server_with_node_spec.rb +44 -0
  57. data/spec/integration_test/definition/server01.rb +20 -0
  58. data/spec/integration_test/definition/server02.rb +16 -0
  59. data/spec/integration_test/definition/task_obj_definition.rb +49 -0
  60. data/spec/manage/manage_spec.rb +33 -0
  61. data/spec/manage/send_signal_spec.rb +39 -0
  62. data/spec/{ssh_shell_spec.rb → manage/ssh_shell_spec.rb} +8 -8
  63. data/spec/node/connection_spec.rb +66 -0
  64. data/spec/node/task_client_spec.rb +212 -0
  65. data/spec/server/acl_file_spec.rb +9 -0
  66. data/spec/{server_check_alive_spec.rb → server/check_alive_spec.rb} +15 -11
  67. data/spec/{data → server/data}/acl.txt +0 -0
  68. data/spec/{history_spec.rb → server/history_spec.rb} +9 -5
  69. data/spec/server/message_spec.rb +195 -0
  70. data/spec/server/node_list_spec.rb +111 -0
  71. data/spec/server/queue_spec.rb +239 -0
  72. data/spec/{server_hook_spec.rb → server/server_hook_spec.rb} +23 -17
  73. data/spec/server/server_spec.rb +89 -0
  74. data/spec/server/transfer_setting_spec.rb +37 -0
  75. data/spec/spec_helper.rb +65 -0
  76. data/spec/task/file_transfer_spec.rb +107 -0
  77. data/spec/{task_generator_spec.rb → task/task_generator_spec.rb} +2 -2
  78. data/spec/{task_spec.rb → task/task_spec.rb} +3 -1
  79. data/spec/utility/argument_spec.rb +39 -0
  80. data/spec/utility/command_line/command_base_spec.rb +33 -0
  81. data/spec/utility/command_line/commands_spec.rb +15 -0
  82. data/spec/utility/misc_spec.rb +77 -0
  83. data/spec/utility/server_define_spec.rb +59 -0
  84. data/spec/utility/temporary_spec.rb +39 -0
  85. metadata +158 -93
  86. data/example/drbqs-manage-test.rb +0 -3
  87. data/example/drbqs-node-test.rb +0 -3
  88. data/example/drbqs-server-test.rb +0 -3
  89. data/lib/drbqs/acl_file.rb +0 -13
  90. data/lib/drbqs/config.rb +0 -98
  91. data/lib/drbqs/connection.rb +0 -67
  92. data/lib/drbqs/history.rb +0 -34
  93. data/lib/drbqs/manage.rb +0 -84
  94. data/lib/drbqs/message.rb +0 -119
  95. data/lib/drbqs/node_list.rb +0 -52
  96. data/lib/drbqs/queue.rb +0 -138
  97. data/lib/drbqs/server_hook.rb +0 -67
  98. data/lib/drbqs/ssh/host.rb +0 -26
  99. data/lib/drbqs/ssh/shell.rb +0 -139
  100. data/lib/drbqs/task_client.rb +0 -86
  101. data/lib/drbqs/utils.rb +0 -19
  102. data/spec/acl_file_spec.rb +0 -9
  103. data/spec/config_spec.rb +0 -14
  104. data/spec/connection_spec.rb +0 -49
  105. data/spec/manage_spec.rb +0 -57
  106. data/spec/message_spec.rb +0 -81
  107. data/spec/node_list_spec.rb +0 -68
  108. data/spec/queue_spec.rb +0 -59
  109. data/spec/server_define_spec.rb +0 -45
  110. data/spec/server_spec.rb +0 -56
  111. data/spec/task_client_spec.rb +0 -53
  112. data/spec/test/test1.rb +0 -13
  113. data/spec/test1_spec.rb +0 -80
  114. data/spec/test2_spec.rb +0 -69
  115. data/spec/transfer_spec.rb +0 -13
@@ -0,0 +1,142 @@
1
+ require 'drbqs/server/node_list'
2
+
3
+ module DRbQS
4
+ class Server
5
+ class Message
6
+ include DRbQS::Misc
7
+
8
+ def initialize(message, logger = DRbQS::LoggerDummy.new)
9
+ @message = message
10
+ @node_list = DRbQS::Server::NodeList.new
11
+ @logger = logger
12
+ end
13
+
14
+ def get_message
15
+ begin
16
+ mes = @message.take([:server, Symbol, nil], 0)
17
+ manage_message(*mes[1..2])
18
+ rescue Rinda::RequestExpiredError
19
+ nil
20
+ end
21
+ end
22
+
23
+ def manage_message(mes, arg)
24
+ @logger.info("Get message") { [mes, arg] }
25
+ case mes
26
+ when :connect
27
+ a = [arg, @node_list.get_new_id(arg)]
28
+ @logger.info("New node") { a }
29
+ @message.write(a)
30
+ when :alive
31
+ @node_list.set_alive(arg)
32
+ when :exit_server
33
+ @logger.info("Get exit message from #{arg.to_s}")
34
+ when :exit_after_task
35
+ @logger.info("Get exit message for node #{arg.to_s} after current task")
36
+ return [mes, arg]
37
+ when :request_status
38
+ @logger.info("Get status request from #{arg.to_s}")
39
+ when :node_error
40
+ @node_list.delete(arg[0])
41
+ @logger.info("Node Error (#{arg[0]})") { arg[1] }
42
+ return [mes, arg[0]]
43
+ else
44
+ @logger.error("Invalid message from #{arg.to_s}")
45
+ return nil
46
+ end
47
+ [mes]
48
+ end
49
+ private :manage_message
50
+
51
+ def check_connection
52
+ deleted = @node_list.delete_not_alive
53
+ @logger.info("IDs of deleted nodes") { deleted } if deleted.size > 0 && @logger
54
+ @node_list.each do |id, str|
55
+ @message.write([id, :alive_p])
56
+ end
57
+ @node_list.set_check_connection
58
+ deleted
59
+ end
60
+
61
+ def send_signal(node_id, signal)
62
+ @message.write([node_id, signal])
63
+ end
64
+ private :send_signal
65
+
66
+ def send_signal_to_all_nodes(signal)
67
+ @node_list.each do |node_id, id_str|
68
+ send_signal(node_id, signal)
69
+ end
70
+ end
71
+ private :send_signal_to_all_nodes
72
+
73
+ # Send all nodes a message to exit.
74
+ def send_exit
75
+ send_signal_to_all_nodes(:exit)
76
+ end
77
+
78
+ # Send all nodes a message to finalize and exit.
79
+ def send_finalization
80
+ send_signal_to_all_nodes(:finalize)
81
+ end
82
+
83
+ def send_exit_after_task(node_id)
84
+ send_signal(node_id, :exit_after_task)
85
+ end
86
+
87
+ def send_status(calculating_task_id)
88
+ s = ''
89
+ @node_list.history.each do |node_id, events|
90
+ if events.size == 0 || events.size > 2
91
+ raise "Invalid history of nodes: #{events.inspect}"
92
+ end
93
+ connect = events[0]
94
+ s << sprintf("%4d %s\t", node_id, connect[2])
95
+ if disconnect = events[1]
96
+ s << "disconnected: (#{time_to_history_string(connect[0])} - #{time_to_history_string(disconnect[0])})\n"
97
+ else
98
+ task_ids = calculating_task_id[node_id].to_a
99
+ s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_history_string(connect[0])})\n"
100
+ end
101
+ end
102
+ begin
103
+ @message.take([:status, nil], 0)
104
+ rescue Rinda::RequestExpiredError
105
+ end
106
+ @message.write([:status, s])
107
+ end
108
+
109
+ def get_all_nodes
110
+ @node_list.list.dup
111
+ end
112
+
113
+ def node_not_exist?
114
+ @node_list.empty?
115
+ end
116
+
117
+ def node_exist?(node_id)
118
+ @node_list.exist?(node_id)
119
+ end
120
+
121
+ def set_special_task(label, task)
122
+ begin
123
+ @message.take([label, nil, Symbol, nil], 0)
124
+ rescue Rinda::RequestExpiredError
125
+ end
126
+ @message.write(task.drb_args(label))
127
+ end
128
+ private :set_special_task
129
+
130
+ # If the task has already set,
131
+ # the method overwrite old task of initialization by new task.
132
+ def set_initialization(task)
133
+ set_special_task(:initialize, task)
134
+ end
135
+
136
+ def set_finalization(task)
137
+ set_special_task(:finalization, task)
138
+ end
139
+ end
140
+
141
+ end
142
+ end
@@ -0,0 +1,59 @@
1
+ require 'drbqs/server/history'
2
+
3
+ module DRbQS
4
+ class Server
5
+ class NodeList
6
+ attr_reader :history, :list
7
+
8
+ def initialize
9
+ @id = 0
10
+ @list = {}
11
+ @check = []
12
+ @history = DRbQS::Server::History.new
13
+ end
14
+
15
+ def get_new_id(id_str)
16
+ @id += 1
17
+ @list[@id] = id_str
18
+ @history.set(@id, :connect, @list[@id])
19
+ @id
20
+ end
21
+
22
+ def each(&block)
23
+ @list.each(&block)
24
+ end
25
+
26
+ def set_check_connection
27
+ @check = @list.keys
28
+ end
29
+
30
+ def delete(id)
31
+ @list.delete(id)
32
+ @history.set(id, :disconnect)
33
+ end
34
+
35
+ def delete_not_alive
36
+ @check.each do |id|
37
+ delete(id)
38
+ end
39
+ deleted = @check
40
+ @check = []
41
+ deleted
42
+ end
43
+
44
+ def set_alive(id)
45
+ @check.delete(id)
46
+ end
47
+
48
+ def empty?
49
+ @list.size == 0
50
+ end
51
+
52
+ def exist?(id)
53
+ @list.find { |a| a[0] == id }
54
+ end
55
+ end
56
+
57
+ end
58
+ end
59
+
@@ -0,0 +1,128 @@
1
+ require 'drbqs/server/history'
2
+
3
+ module DRbQS
4
+
5
+ class Server
6
+ class Queue
7
+ attr_reader :calculating, :history
8
+
9
+ def initialize(queue, result, logger = DRbQS::LoggerDummy.new)
10
+ @queue = queue
11
+ @result = result
12
+ @task_id = 0
13
+ @cache = {}
14
+ @calculating = Hash.new { |hash, key| hash[key] = Array.new }
15
+ @history = DRbQS::Server::History.new
16
+ @logger = logger
17
+ end
18
+
19
+ def queue_task(task_id)
20
+ @queue.write(@cache[task_id].drb_args(task_id))
21
+ end
22
+ private :queue_task
23
+
24
+ # &hook take two arguments: a DRbQS::Server::Queue object and a result of task.
25
+ # Return task ID (for debug).
26
+ def add(task)
27
+ @task_id += 1
28
+ @logger.info("New task: #{@task_id}")
29
+ @cache[@task_id] = task
30
+ queue_task(@task_id)
31
+ @history.set(@task_id, :add)
32
+ @task_id
33
+ end
34
+
35
+ def get_accept_signal
36
+ count = 0
37
+ begin
38
+ loop do
39
+ sym, task_id, node_id = @result.take([:accept, Fixnum, Fixnum], 0)
40
+ count += 1
41
+ @calculating[node_id] << task_id
42
+ @history.set(task_id, :calculate, node_id)
43
+ @logger.info("Accept: task #{task_id} by node #{node_id}.")
44
+ end
45
+ rescue Rinda::RequestExpiredError
46
+ @logger.debug("Accept: #{count} signals.")
47
+ end
48
+ count
49
+ end
50
+
51
+ def requeue_for_deleted_node_id(deleted)
52
+ deleted.each do |node_id|
53
+ if task_id_ary = @calculating[node_id]
54
+ task_id_ary.each do |task_id|
55
+ queue_task(task_id)
56
+ @history.set(task_id, :requeue)
57
+ @logger.info("Requeue: task #{task_id}.")
58
+ end
59
+ @calculating.delete(node_id)
60
+ end
61
+ end
62
+ end
63
+
64
+ def delete_task_id(node_id, task_id)
65
+ unless @calculating[node_id].delete(task_id)
66
+ @logger.error("Task #{task_id} does not exist in list of calculating tasks.")
67
+ end
68
+ if ary = @calculating.find { |k, v| v.include?(task_id) }
69
+ @logger.error("Node #{ary[0]} is calculating task #{task_id}, too.")
70
+ end
71
+ end
72
+ private :delete_task_id
73
+
74
+ def exec_task_hook(main_server, task_id, result)
75
+ if task = @cache.delete(task_id)
76
+ if task.exec_hook(main_server, result)
77
+ @history.set(task_id, :hook)
78
+ end
79
+ true
80
+ else
81
+ @logger.error("Task #{task_id} is not cached.")
82
+ false
83
+ end
84
+ end
85
+
86
+ def get_result(main_server)
87
+ count = 0
88
+ begin
89
+ loop do
90
+ get_accept_signal
91
+ sym, task_id, node_id, result = @result.take([:result, Fixnum, Fixnum, nil], 0)
92
+ count += 1
93
+ @history.set(task_id, :result, node_id)
94
+ @logger.info("Get: result of #{task_id} from node #{node_id}.")
95
+ delete_task_id(node_id, task_id)
96
+ exec_task_hook(main_server, task_id, result)
97
+ end
98
+ rescue Rinda::RequestExpiredError
99
+ @logger.debug("Get: #{count} results.")
100
+ end
101
+ count
102
+ end
103
+
104
+ def calculating_task_number
105
+ @calculating.inject(0) { |s, key_val| s + key_val[1].size }
106
+ end
107
+
108
+ # If queue is empty, that is, there is no tasks to calculate next,
109
+ # this method returns true. Otherwise, false.
110
+ # Even if there are calculating tasks,
111
+ # the method can return true.
112
+ def empty?
113
+ @cache.size - calculating_task_number == 0
114
+ end
115
+
116
+ # If there are no tasks in queue and calculating,
117
+ # return true. Otherwise, false.
118
+ def finished?
119
+ @cache.size == 0
120
+ end
121
+
122
+ def all_logs
123
+ @history.log_strings
124
+ end
125
+ end
126
+
127
+ end
128
+ end
@@ -1,28 +1,12 @@
1
- require 'drbqs/message'
2
- require 'drbqs/queue'
3
- require 'drbqs/acl_file'
4
- require 'drbqs/server_hook'
1
+ require 'drbqs/task/task'
2
+ require 'drbqs/server/transfer_setting'
3
+ require 'drbqs/server/check_alive'
4
+ require 'drbqs/server/message'
5
+ require 'drbqs/server/queue'
6
+ require 'drbqs/server/acl_file'
7
+ require 'drbqs/server/server_hook'
5
8
 
6
9
  module DRbQS
7
- class CheckAlive
8
- DEFAULT_INTERVAL_TIME = 300
9
-
10
- def initialize(interval)
11
- @interval = interval || DEFAULT_INTERVAL_TIME
12
- if !(Numeric === @interval) || @interval < 0
13
- raise ArgumentError, "Invalid interval time."
14
- end
15
- @last = Time.now
16
- end
17
-
18
- def significant_interval?
19
- (Time.now - @last) >= @interval
20
- end
21
-
22
- def set_checking
23
- @last = Time.now
24
- end
25
- end
26
10
 
27
11
  # When we set both empty_queue_hook and task_generator,
28
12
  # empty_queue_hook is prior to task_generator.
@@ -31,10 +15,13 @@ module DRbQS
31
15
  WAIT_TIME_NODE_FINALIZE = 10
32
16
  WAIT_TIME_NEW_RESULT = 1
33
17
 
34
- attr_reader :queue
18
+ attr_reader :queue, :uri
35
19
 
36
20
  # :port
37
21
  # Set the port of server.
22
+ # :unix
23
+ # Set the path of unix domain socket.
24
+ # If :port is specified, :port is preceded.
38
25
  # :acl
39
26
  # Set the ACL instance.
40
27
  # :log_file
@@ -47,51 +34,45 @@ module DRbQS
47
34
  # Exit programs in finish_hook.
48
35
  # :signal_trap
49
36
  # Set trapping signal.
50
- # :scp_user
51
- # Set user of scp.
52
- # :scp_host
53
- # Set host of scp.
37
+ # :sftp_user
38
+ # Set user of sftp.
39
+ # :sftp_host
40
+ # Set host of sftp.
54
41
  # :file_directory
55
42
  # Set the setting of file directory.
56
43
  def initialize(opts = {})
57
- @port = opts[:port] || ROOT_DEFAULT_PORT
44
+ @uri = DRbQS::Misc.create_uri(opts)
58
45
  @acl = acl_init(opts[:acl])
46
+ @key = DRbQS::Misc.random_key + sprintf("_%d", Time.now.to_i)
59
47
  @ts = {
60
48
  :message => Rinda::TupleSpace.new,
61
49
  :queue => Rinda::TupleSpace.new,
62
50
  :result => Rinda::TupleSpace.new,
51
+ :key => @key,
63
52
  :transfer => nil
64
53
  }
65
- @logger = DRbQS::Utils.create_logger(opts[:log_file], opts[:log_level])
66
- @message = MessageServer.new(@ts[:message], @logger)
67
- @queue= QueueServer.new(@ts[:queue], @ts[:result], @logger)
68
- @check_alive = CheckAlive.new(opts[:check_alive])
54
+ @logger = DRbQS::Misc.create_logger(opts[:log_file], opts[:log_level])
55
+ @message = DRbQS::Server::Message.new(@ts[:message], @logger)
56
+ @queue= DRbQS::Server::Queue.new(@ts[:queue], @ts[:result], @logger)
57
+ @check_alive = DRbQS::Server::CheckAlive.new(opts[:check_alive])
69
58
  @task_generator = []
70
59
  hook_init(opts[:finish_exit])
71
60
  set_signal_trap if opts[:signal_trap]
72
61
  @finalization_task = nil
73
- @transfer_setting = get_transfer_setting(opts[:scp_host], opts[:scp_user], opts[:file_directory])
62
+ @transfer_setting = DRbQS::Server::TransferSetting.new(opts[:sftp_host], opts[:sftp_user], opts[:file_directory])
63
+ @config = DRbQS::Config.new
74
64
  end
75
65
 
76
66
  def transfer_directory
77
67
  @ts[:transfer] && @ts[:transfer].directory
78
68
  end
79
69
 
80
- def get_transfer_setting(host, user, directory)
81
- setting = { :directory => directory, :user => user, :host => host, :set => true }
82
- if host || user || directory
83
- setting[:set] = true
84
- end
85
- setting
86
- end
87
- private :get_transfer_setting
88
-
89
70
  def acl_init(acl_arg)
90
71
  case acl_arg
91
72
  when Array
92
73
  ACL.new(acl_arg)
93
74
  when String
94
- ACLFile.load(acl_arg)
75
+ DRbQS::Server::ACLFile.load(acl_arg)
95
76
  else
96
77
  nil
97
78
  end
@@ -99,24 +80,27 @@ module DRbQS
99
80
  private :acl_init
100
81
 
101
82
  def hook_init(finish_exit)
102
- @hook = DRbQS::ServerHook.new
83
+ @hook = DRbQS::Server::Hook.new
103
84
  @hook.set_finish_exit { self.exit } if finish_exit
104
85
  end
105
86
  private :hook_init
106
87
 
88
+ def server_data
89
+ { :pid => Process.pid, :key => @key }
90
+ end
91
+ private :server_data
92
+
107
93
  def start
108
- if @transfer_setting[:set] && @transfer_setting[:directory] && !@ts[:transfer]
109
- set_file_transfer(@transfer_setting[:directory])
110
- end
94
+ set_file_transfer(nil)
111
95
  DRb.install_acl(@acl) if @acl
112
- uri = "druby://:#{@port}"
113
- DRb.start_service(uri, @ts)
114
- @logger.info("Start DRb service") { uri } if @logger
96
+ DRb.start_service(@uri, @ts)
97
+ @config.list.server.save(@uri, server_data)
98
+ @logger.info("Start DRb service") { @uri }
115
99
  end
116
100
 
117
101
  def check_connection(force = nil)
118
102
  if force || @check_alive.significant_interval?
119
- @logger.info("Check connections.") if @logger
103
+ @logger.info("Check connections.")
120
104
  deleted_node_ids = @message.check_connection
121
105
  @queue.requeue_for_deleted_node_id(deleted_node_ids)
122
106
  @check_alive.set_checking
@@ -137,11 +121,12 @@ module DRbQS
137
121
  if @task_generator.size > 0 && @queue.empty?
138
122
  if tasks = @task_generator[0].new_tasks
139
123
  tasks.each { |t| @queue.add(t) }
140
- @logger.info("Generator add #{tasks.size} tasks.") if @logger
124
+ @logger.info("Generator add #{tasks.size} tasks.")
141
125
  else
142
126
  @task_generator.delete_at(0)
143
- @logger.info("Generator creates all tasks and then has been deleted.") if @logger
127
+ @logger.info("Generator creates all tasks and then has been deleted.")
144
128
  if @task_generator.size > 0
129
+ first_task_generator_init
145
130
  add_tasks_from_generator
146
131
  end
147
132
  end
@@ -155,6 +140,7 @@ module DRbQS
155
140
 
156
141
  def set_finalization_task(task)
157
142
  @finalization_task = task
143
+ @message.set_finalization(@finalization_task)
158
144
  end
159
145
 
160
146
  # +key+ is :empty_queue or :finish_exit.
@@ -168,14 +154,14 @@ module DRbQS
168
154
  end
169
155
 
170
156
  def exec_finish_hook
171
- @logger.info("Execute finish hook.") if @logger
157
+ @logger.info("Execute finish hook.")
172
158
  @hook.exec(:finish, self)
173
159
  end
174
160
  private :exec_finish_hook
175
161
 
176
162
  def exec_hook
177
163
  if @queue.empty?
178
- @logger.info("Execute empty queue hook.") if @logger
164
+ @logger.info("Execute empty queue hook.")
179
165
  @hook.exec(:empty_queue, self)
180
166
  end
181
167
  if !generator_waiting? || @queue.finished?
@@ -189,7 +175,7 @@ module DRbQS
189
175
 
190
176
  def exit
191
177
  if @finalization_task
192
- @message.send_finalization(@finalization_task)
178
+ @message.send_finalization
193
179
  wait_time = WAIT_TIME_NODE_FINALIZE
194
180
  else
195
181
  @message.send_exit
@@ -199,7 +185,8 @@ module DRbQS
199
185
  sleep(wait_time)
200
186
  check_connection(true)
201
187
  end
202
- @logger.info("History of tasks") { "\n" + @queue.all_logs } if @logger
188
+ @logger.info("History of tasks") { "\n" + @queue.all_logs }
189
+ @config.list.server.delete(@uri)
203
190
  Kernel.exit
204
191
  end
205
192
 
@@ -210,10 +197,10 @@ module DRbQS
210
197
  end
211
198
 
212
199
  def set_file_transfer(directory, opts = {})
213
- user = opts[:user] || @transfer_setting[:user] || ENV['USER']
214
- host = opts[:host] || @transfer_setting[:host] || 'localhost'
215
- @ts[:transfer] = DRbQS::Transfer.new(user, host, directory)
216
- @logger.info("File transfer") { @ts[:transfer].information } if @logger
200
+ if transfer = @transfer_setting.create(directory, opts)
201
+ @ts[:transfer] = transfer
202
+ @logger.info("File transfer") { transfer.information }
203
+ end
217
204
  end
218
205
 
219
206
  def check_message
@@ -224,6 +211,11 @@ module DRbQS
224
211
  self.exit
225
212
  when :request_status
226
213
  @message.send_status(@queue.calculating)
214
+ when :exit_after_task
215
+ node_id = arg
216
+ if @message.node_exist?(node_id)
217
+ @message.send_exit_after_task(node_id)
218
+ end
227
219
  when :node_error
228
220
  @queue.get_accept_signal
229
221
  @queue.requeue_for_deleted_node_id([arg])
@@ -232,19 +224,19 @@ module DRbQS
232
224
  end
233
225
  private :check_message
234
226
 
235
- def task_generator_init
236
- @task_generator.each { |tgen| tgen.init }
227
+ def first_task_generator_init
228
+ @task_generator[0].init if @task_generator[0]
237
229
  end
238
- private :task_generator_init
230
+ private :first_task_generator_init
239
231
 
240
232
  def wait
241
- task_generator_init
233
+ first_task_generator_init
242
234
  loop do
243
235
  check_message
244
236
  check_connection
245
237
  count_results = @queue.get_result(self)
246
238
  exec_hook
247
- @logger.debug("Calculating tasks: #{@queue.calculating_task_number}") if @logger
239
+ @logger.debug("Calculating tasks: #{@queue.calculating_task_number}")
248
240
  if count_results <= 1
249
241
  sleep(WAIT_TIME_NEW_RESULT)
250
242
  end
@@ -267,11 +259,11 @@ module DRbQS
267
259
  private :finish_profile
268
260
 
269
261
  def test_exec(opts = {})
270
- task_generator_init
271
- dummy_client = DRbQS::Client.new(nil, :log_file => $stdout, :log_level => opts[:log_level])
272
- dummy_task_client = DRbQS::TaskClient.new(nil, @ts[:queue], nil)
262
+ first_task_generator_init
263
+ dummy_node = DRbQS::Node.new(nil, :log_file => $stdout, :log_level => opts[:log_level])
264
+ dummy_task_client = DRbQS::Node::TaskClient.new(nil, @ts[:queue], nil)
273
265
  if @ts[:transfer]
274
- dummy_client.instance_variable_set(:@transfer, DRbQS::TransferTest.new(@ts[:transfer].directory))
266
+ dummy_node.instance_variable_set(:@transfer, DRbQS::TransferClient::Local.new(@ts[:transfer].directory))
275
267
  end
276
268
  num = 0
277
269
  start_profile if opts[:profile]
@@ -279,7 +271,7 @@ module DRbQS
279
271
  exec_hook
280
272
  if ary = dummy_task_client.get_task
281
273
  task_id, marshal_obj, method_sym, args = ary
282
- result = dummy_client.instance_eval { execute_task(marshal_obj, method_sym, args) }
274
+ result = dummy_node.instance_eval { execute_task(marshal_obj, method_sym, args) }
283
275
  @queue.exec_task_hook(self, task_id, result)
284
276
  end
285
277
  num += 1
@@ -290,15 +282,15 @@ module DRbQS
290
282
  finish_profile if opts[:profile]
291
283
  if @finalization_task
292
284
  args = @finalization_task.drb_args(nil)[1..-1]
293
- dummy_client.instance_eval { execute_task(*args) }
285
+ dummy_node.instance_eval { execute_task(*args) }
294
286
  end
295
287
  exec_finish_hook
296
288
  end
297
289
 
298
290
  def test_task_generator(opts = {})
299
- task_generator_init
300
291
  @task_generator.each_with_index do |t, i|
301
292
  puts "Test task generator [#{i}]"
293
+ t.init
302
294
  set_num, task_num = t.debug_all_tasks(opts)
303
295
  puts "Create: task sets #{set_num}, all tasks #{task_num}"
304
296
  end
@@ -0,0 +1,72 @@
1
+ module DRbQS
2
+ class Server
3
+ class Hook
4
+ def initialize
5
+ @hook = Hash.new { |h, k| h[k] = Array.new }
6
+ @argument_number = {}
7
+ @finish_exit = nil
8
+ set_argument_number(:empty_queue, 1)
9
+ set_argument_number(:finish, 1)
10
+ end
11
+
12
+ def set_argument_number(key, num)
13
+ @argument_number[key] = num
14
+ end
15
+ private :set_argument_number
16
+
17
+ def create_proc_name(key)
18
+ name = "#{key.to_s}#{rand(1000)}"
19
+ if @hook.has_key?(name)
20
+ create_proc_name(key)
21
+ else
22
+ name
23
+ end
24
+ end
25
+ private :create_proc_name
26
+
27
+ def add(key, name = nil, &block)
28
+ if (n = @argument_number[key]) && (block.arity != n)
29
+ raise ArgumentError, "Invalid argument number of hook of #{key.inspect}."
30
+ end
31
+ name ||= create_proc_name(key)
32
+ @hook[key] << [name, block]
33
+ name
34
+ end
35
+
36
+ def delete(key, name = nil)
37
+ if name
38
+ @hook[key].delete_if { |ary| ary[0] == name }
39
+ else
40
+ @hook[key].clear
41
+ end
42
+ end
43
+
44
+ def specific_proc(key)
45
+ case key
46
+ when :finish
47
+ @finish_exit.call if @finish_exit
48
+ end
49
+ end
50
+ private :specific_proc
51
+
52
+ def hook_names(key)
53
+ @hook[key].map { |a| a[0] }
54
+ end
55
+
56
+ def exec(key, *args)
57
+ @hook[key].each do |ary|
58
+ ary[1].call(*args)
59
+ end
60
+ specific_proc(key)
61
+ end
62
+
63
+ def set_finish_exit(&block)
64
+ @finish_exit = block
65
+ end
66
+
67
+ def number_of_hook(key)
68
+ @hook[key].size
69
+ end
70
+ end
71
+ end
72
+ end