drbqs 0.0.17 → 0.0.18

data/lib/drbqs/node/state.rb

@@ -2,78 +2,142 @@ module DRbQS
   class Node
     class State
       # Value of state is :sleep, :wait, or :calculate.
-      attr_reader :state
+      attr_reader :calculating_task
 
-      ALL_STATES = [:sleep, :wait, :calculate]
+      ALL_STATES = [:sleep, :wait, :calculate, :exit]
       DEFAULT_SLEEP_TIME = 300
       LOADAVG_PATH = '/proc/loadavg'
 
-      def initialize(state, opts = {})
-        @state = state
-        @sleep_at_calculated = nil
+      def initialize(state_init, process_number, opts = {})
+        @process_number = process_number
+        @process_state = {}
+        @process_number.times do |i|
+          @process_state[i] = state_init
+        end
+        @calculating_task = {}
+        @state_after_task = nil
+
+
         @load_average_threshold = opts[:max_loadavg]
         @sleep_time = opts[:sleep_time] || DEFAULT_SLEEP_TIME
         @auto_wakeup = nil
       end
 
-      def change(state)
-        unless ALL_STATES.include?(state)
-          raise ArgumentError, "Invalid state of node '#{state}'."
-        end
-        @state = state
+      def get_state(wid)
+        @process_state[wid]
       end
 
-      def calculate?
-        @state == :calculate
+      def each_worker_id(&block)
+        if block_given?
+          @process_state.each do |key, val|
+            yield(key)
+          end
+        else
+          to_enum(:each_worker_id)
+        end
       end
 
-      def rest?
-        !calculate?
+      def waiting_worker_id
+        ary = []
+        @process_state.each do |wid, state|
+          if state == :wait
+            ary << wid
+          end
+        end
+        ary
       end
 
-      def stop?
-        @state == :sleep
+      def request_task_number
+        waiting = @process_state.select do |wid, state|
+          state == :wait
+        end
+        waiting.size
       end
 
       def request?
-        @state == :wait
+        request_task_number > 0
       end
 
-      def change_to_wait
-        unless calculate?
-          change(:wait)
+      def all_workers_waiting?
+        each_worker_id.all? do |wid|
+          st = get_state(wid)
+          st == :wait || st == :exit
         end
       end
 
-      def change_to_sleep
-        if calculate?
-          @sleep_at_calculated = true
-        else
-          change(:sleep)
+      def set_calculating_task(wid, task_id)
+        @calculating_task[task_id] = wid
+        change(wid, :calculate)
+      end
+
+      def set_exit_after_task
+        @state_after_task = :exit
+        each_worker_id do |wid|
+          st = get_state(wid)
+          if (st == :wait) && (st == :sleep)
+            change(wid, :exit)
+          end
         end
       end
 
-      def change_to_calculate
-        change(:calculate)
+      def set_finish_of_task(sent_task_id)
+        sent_task_id.each do |task_id|
+          if wid = @calculating_task.delete(task_id)
+            case @state_after_task
+            when :exit
+              @process_state[wid] = :exit
+            when :sleep
+              @process_state[wid] = :sleep
+            else
+              @process_state[wid] = :wait
+            end
+          end
+        end
       end
 
-      def change_to_finish_calculating
-        if @sleep_at_calculated
-          change(:sleep)
-        else
-          change(:wait)
+      def change(proc_id, state)
+        unless ALL_STATES.include?(state)
+          raise ArgumentError, "Invalid state of node '#{state}'."
+        end
+        @process_state[proc_id] = state
+      end
+
+      def wakeup_sleeping_worker
+        each_worker_id do |wid|
+          if get_state(wid) == :sleep
+            change(wid, :wait)
+          end
+        end
+        @state_after_task = nil
+      end
+
+      def change_to_sleep
+        each_worker_id do |wid|
+          st = get_state(wid)
+          if st == :calculate
+            @state_after_task = :sleep
+          elsif st != :exit
+            change(wid, :sleep)
+          end
         end
-        @sleep_at_calculated = nil
       end
 
       def sleep_with_auto_wakeup
-        change(:sleep)
+        each_worker_id do |wid|
+          if get_state(wid) == :wait
+            change(wid, :sleep)
+          end
+        end
         @auto_wakeup = Time.now + @sleep_time
       end
 
       def wakeup_automatically_for_unbusy_system
         if @auto_wakeup && Time.now > @auto_wakeup && !system_busy?
-          change(:wait)
+          each_worker_id do |wid|
+            if get_state(wid) == :sleep
+              change(wid, :wait)
+            end
+          end
           @auto_wakeup = nil
           return true
         end
@@ -96,6 +160,7 @@ module DRbQS
         end
         nil
       end
+      private :system_busy?
 
       def change_to_sleep_for_busy_system
         if system_busy?

data/lib/drbqs/node/task_client.rb

@@ -1,23 +1,18 @@
 module DRbQS
   class Node
     class TaskClient
-      attr_reader :node_number, :calculating_task
+      attr_reader :node_number, :group
 
-      def initialize(node_number, queue, result, logger = DRbQS::Misc::LoggerDummy.new)
+      def initialize(node_number, queue, result, group, logger = DRbQS::Misc::LoggerDummy.new)
         @node_number = node_number
         @queue = queue
         @result = result
-        @calculating_task = nil
-        @exit_after_task = nil
         @task_queue = Queue.new
         @result_queue = Queue.new
+        @group = group || []
         @logger = logger
       end
 
-      def calculating?
-        !!@calculating_task
-      end
-
       def task_empty?
         @task_queue.empty?
       end
@@ -31,57 +26,75 @@ module DRbQS
       end
       private :dequeue_result
 
-      def queue_task(task_id, ary)
-        @calculating_task = task_id
+      # @param [Array] ary An array is [task_id, obj, method_name, args]
+      def queue_task(ary)
         @task_queue.enq(ary)
       end
 
+      # @return [nil,Array] If @task_queue is empty then return nil.
+      #   Otherwise, an array [task_id, obj, method_name, args] is returned.
       def dequeue_task
-        @task_queue.deq
+        if @task_queue.empty?
+          nil
+        else
+          @task_queue.deq
+        end
       end
 
-      def get_task
+      def get_task_by_group(grp)
         begin
-          @queue.take([Fixnum, nil, Symbol, nil], 0)
+          @queue.take([grp, Fixnum, nil, Symbol, nil], 0)[1..-1]
         rescue Rinda::RequestExpiredError
           nil
         end
       end
 
-      def set_exit_after_task
-        @exit_after_task = true
+      def get_task
+        @group.each do |grp|
+          if task = get_task_by_group(grp)
+            return task
+          end
+        end
+        get_task_by_group(DRbQS::Task::DEFAULT_GROUP)
       end
 
-      def add_new_task
-        if !@calculating_task && !@exit_after_task && (ary = get_task)
-          task_id, obj, method_sym, args = ary
-          @logger.info("Send accept signal: node #{@node_number} caluclating #{task_id}")
-          @result.write([:accept, task_id, @node_number])
-          queue_task(task_id, [obj, method_sym, args])
-          return true
+      def add_new_task(num)
+        get_task_id = []
+        num.times do |i|
+          if ary = get_task
+            task_id = ary[0]
+            @logger.info("Send accept signal: node #{@node_number} caluclating #{task_id}")
+            @result.write([:accept, task_id, @node_number])
+            queue_task(ary)
+            get_task_id << task_id
+          else
+            break
+          end
         end
-        nil
+        get_task_id.empty? ? nil: get_task_id
       end
 
-      # If the method return true, a node should finilize and exit.
+      # Return an array of task ID that is sent to the server.
       def send_result
-        if !result_empty?
-          result = dequeue_result
-          @logger.info("Send result: #{@calculating_task}") { result.inspect }
-          @result.write([:result, @calculating_task, @node_number, result])
-          @calculating_task = nil
+        sent_task_id = []
+        while !result_empty?
+          task_id, result = dequeue_result
+          @logger.info("Send result: #{task_id}") { result.inspect }
+          @result.write([:result, task_id, @node_number, result])
+          sent_task_id << task_id
         end
-        !@calculating_task && @exit_after_task
+        sent_task_id.empty? ? nil : sent_task_id
       end
 
-      def queue_result(result)
-        @result_queue.enq(result)
+      def queue_result(task_id, result)
+        @result_queue.enq([task_id, result])
       end
 
       def dump_result_queue
         results = []
         while !result_empty?
-          results << dequeue_result
+          task_id, res = dequeue_result
+          results << res
         end
         if results.size > 0
           Marshal.dump(results)

data/lib/drbqs/server/message.rb

@@ -20,6 +20,8 @@ module DRbQS
       # * [:wake_node, node_id]
       # * [:sleep_node, node_id]
       # * [:node_error, [node_id, error_message]]
+      # * [:initialize, Array]
+      # * [:finalize, Array]
       # 
       # @return [Array] Message array
       def get_message
@@ -148,23 +150,27 @@ module DRbQS
         @node_list.exist?(node_id)
       end
 
-      def set_special_task(label, task)
+      def set_special_task(label, *tasks)
+        task_ary = []
         begin
-          @message.take([label, nil, Symbol, nil], 0)
+          task_ary = @message.take([label, Array], 0)[1]
         rescue Rinda::RequestExpiredError
         end
-        @message.write(task.drb_args(label))
+        tasks.each do |task|
+          task_ary << task.simple_drb_args
+        end
+        @message.write([label, task_ary])
       end
       private :set_special_task
 
       # If the task has already set,
       # the method overwrite old task of initialization by new task.
-      def set_initialization(task)
-        set_special_task(:initialize, task)
+      def set_initialization_tasks(task_ary)
+        set_special_task(:initialize, *task_ary)
       end
 
-      def set_finalization(task)
-        set_special_task(:finalization, task)
+      def set_finalization_tasks(task_ary)
+        set_special_task(:finalize, *task_ary)
       end
 
       def shutdown_unused_nodes(calculating_nodes)

data/lib/drbqs/server/server.rb

@@ -18,18 +18,33 @@ module DRbQS
     attr_reader :queue, :uri
 
     # @param [Hash] opts The options of server
-    # @option opts [Hash] :port Set the port of server.
-    # @option opts [Hash] :unix Set the path of unix domain socket. If :port is specified then :port is preceded.
-    # @option opts [Hash] :acl Set the ACL instance.
-    # @option opts [Hash] :log_file Set the path of log files.
-    # @option opts [Hash] :log_level Set the level of logging.
-    # @option opts [Hash] :check_alive Set the time interval of checking alive nodes.
-    # @option opts [Hash] :not_exit Not exit programs when all tasks are finished.
-    # @option opts [Hash] :shutdown_unused_nodes Shutdown unused nodes.
-    # @option opts [Hash] :signal_trap Set trapping signal. Default is true.
-    # @option opts [Hash] :sftp_user Set user of sftp.
-    # @option opts [Hash] :sftp_host Set host of sftp.
-    # @option opts [Hash] :file_directory Set the directory for nodes to send files.
+    # @option opts [Fixnum] :port Set the port of server.
+    # @option opts [String] :unix Set the path of unix domain socket. If :port is specified then :port is preceded.
+    # @option opts [Array] :acl Set the array of ACL.
+    # @option opts [String] :acl Set the file path of ACL.
+    # @option opts [String] :log_file Set the path of log files.
+    # @option opts [Fixnum] :log_level Set the level of logging.
+    # @option opts [Fixnum] :check_alive Set the time interval of checking alive nodes.
+    # @option opts [Boolean] :not_exit Not exit programs when all tasks are finished.
+    # @option opts [Boolean] :shutdown_unused_nodes Shutdown unused nodes.
+    # @option opts [Boolean] :signal_trap Set trapping signal. Default is true.
+    # @option opts [String] :sftp_user Set user of sftp.
+    # @option opts [String] :sftp_host Set host of sftp.
+    # @option opts [String] :file_directory Set the directory for nodes to send files.
+    # 
+    # :nodoc:
+    # * Note of tuple spaces
+    # @ts[:message]
+    #  - used in node/connection.rb
+    #  - some messages to both server and node
+    #  - special tasks from server to nodes
+    # @ts[:queue]
+    #  - used in node/task_client.rb
+    #  - tasks from server to nodes
+    # @ts[:result]
+    #  - used in node/task_client.rb
+    #  - accept signal from nodes
+    #  - results from nodes
     def initialize(opts = {})
       @uri = DRbQS::Misc.create_uri(opts)
       @acl = acl_init(opts[:acl])
@@ -48,7 +63,7 @@ module DRbQS
       @task_generator = []
       hook_init(!opts[:not_exit], opts[:shutdown_unused_nodes])
       set_signal_trap if !opts.has_key?(:signal_trap) || opts[:signal_trap]
-      @finalization_task = nil
+      @finalization_task = []
       @data_storage = []
       @transfer_setting = DRbQS::Server::TransferSetting.new(opts[:sftp_host], opts[:sftp_user], opts[:file_directory])
       @config = DRbQS::Config.new
@@ -107,7 +122,9 @@ module DRbQS
     end
 
     # Create new task generator and add it.
-    # The arguments are same as {DRbQS::Task::Generator#set}.
+    # @param [Hash] opts An argument is same as {DRbQS::Task::Generator#set}
+    # @yield [tgen] Block is same as {DRbQS::Task::Generator#set}
+    # @yieldparam [DRbQS::TaskGenerator] tgen Task generator to add to the server
     def task_generator(opts = {}, &block)
       gen = DRbQS::Task::Generator.new
       gen.set(opts, &block)
@@ -145,29 +162,30 @@ module DRbQS
     private :all_tasks_assigned?
 
     # @param [DRbQS::task] task
-    def set_initialization_task(task)
-      @message.set_initialization(task)
+    def set_initialization_task(*tasks)
+      @message.set_initialization_tasks(tasks)
     end
 
     # @param [DRbQS::task] task
-    def set_finalization_task(task)
-      @finalization_task = task
-      @message.set_finalization(@finalization_task)
+    def set_finalization_task(*tasks)
+      @finalization_task.concat(tasks)
     end
 
     # Set a hook of server.
     # @note When we set both :empty_queue and task generators,
     #  hook of :empty_queue is prior to task generators.
     # @param [:empty_queue,:process_data,:finish] key Set the type of hook.
-    # @param [String] name Name of the hook. If the value is nil then the name is automatically created.
+    # @option opts [Fixnum] :repeat If we execute the hook specified times then the hook is deleted.
+    #   If the value is nil, the hook is repeated without limit.
+    # @option opts [String] :name Name of the hook. If the value is nil then the name is automatically created.
     # @param [Proc] &block block is obligatory and takes server itself as an argument.
-    def add_hook(key, name = nil, &block)
+    def add_hook(key, opts = {}, &block)
       if key == :process_data
         if @hook.number_of_hook(:process_data) != 0
           raise "Hook :process_data has already set."
         end
       end
-      @hook.add(key, name, &block)
+      @hook.add(key, opts, &block)
     end
 
     # @param [:empty_queue,:process_data,:finish] key Set the type of hook.
@@ -238,7 +256,8 @@ module DRbQS
     private :shutdown_unused_nodes
 
     def exit
-      if @finalization_task
+      if !@finalization_task.empty?
+        @message.set_finalization_tasks(@finalization_task)
         @message.send_finalization
         wait_time = WAIT_TIME_NODE_FINALIZE
       else
@@ -263,8 +282,8 @@ module DRbQS
 
     # @param [String] directory Set the directory to save files from nodes.
     # @param [Hash] opts The options for SFTP.
-    # @option opts [Symbol] :host Hostname for SFTP.
-    # @option opts [Symbol] :user User name for SFTP.
+    # @option opts [String] :host Hostname for SFTP.
+    # @option opts [String] :user User name for SFTP.
     def set_file_transfer(directory, opts = {})
       if @transfer_setting.setup_server(directory, opts)
         @ts[:transfer] = @transfer_setting
@@ -295,6 +314,7 @@ module DRbQS
     private :process_data
 
     def send_status_for_request
+      task_message = []
       messages = @queue.calculating_task_message
       s = ''
       @message.each_node_history do|node_id, events|
@@ -313,10 +333,9 @@ module DRbQS
             task_ids = @queue.calculating[node_id].to_a
             s << "task: "
             if messages[node_id]
-              s << messages[node_id].map do |task_id, mes|
-                calc_task = task_id.to_s
-                calc_task << ": " << mes.to_s if mes
-                calc_task
+              s << messages[node_id].map do |ary|
+                task_message << ary
+                ary[0].to_s
               end.join(', ')
             else
               s << "none"
@@ -327,6 +346,15 @@ module DRbQS
       end
       s << "  none\n" if s.size == 0
       s = "Nodes:\n" << s
+      unless task_message.empty?
+        s << "Tasks:\n"
+        task_message.sort_by! do |task_id, mes|
+          task_id
+        end
+        task_message.each do |task_id, mes|
+          s << sprintf("%4d: %s\n", task_id, (mes ? mes.to_s : ''))
+        end
+      end
       s << "Server:\n"
       s << "  calculating tasks: #{@queue.calculating_task_number}\n"
       s << "  finished tasks   : #{@queue.finished_task_number}\n"