drbqs 0.0.17 → 0.0.18

data/docs/FormatExecute.md

@@ -11,7 +11,9 @@ drbqs-execute execute a server of which uri is made from hostname and port.
 Moreover, drbqs-execute make nodes connecting to the uri of server.
 The server and nodes can be over SSH.
 
-## Example: execute.rb
+## Example
+
+### execute.rb
 
     #!/usr/bin/env drbqs-execute
     # -*-ruby-*-
@@ -55,7 +57,21 @@ The server and nodes can be over SSH.
     node :even, group: [:node02, :node04, :node06]
     node :odd, group: [:node01, :node03, :node05]
 
-## Help message
+### Execution
+
+In the above example, there is the shebang line
+
+    #!/usr/bin/env drbqs-execute
+
+and therefore we can execute by
+
+    ./execute.rb
+
+If there is no shebang line, we type
+
+    drbqs-execute execute.rb
+
+### Help message
 
 If we run the following command
 
@@ -63,6 +79,32 @@ If we run the following command
 
 then help message of server.rb is displayed in addition to that of drbqs-execute.
 
+### Information of server and nodes
+
+The command
+
+    drbqs-execute -i
+
+shows information of server and nodes.
+The output is
+
+    Server:
+     * server1    ssh
+       local      local
+    Node:
+     - node_base  ssh,template
+       node01     ssh
+       node02     ssh
+       node03     ssh
+       node04     ssh
+       node05     ssh
+       node06     ssh
+     - even       group: node02,node04,node06
+     - odd        group: node01,node03,node05
+    Port: 12345
+
+The character "*"  means default and "-" means virtual nodes (template or group).
+
 ## Methods
 
 The following methods are available.

data/example/group/execute.rb

@@ -0,0 +1,19 @@
+default :server => :server_local, :port => 13789, :log => '/tmp/drbqs_execute'
+
+current_dir = File.expand_path(File.dirname(__FILE__))
+
+usage :message => "Calculate sum of numbers", :server => File.join(current_dir, 'server.rb')
+
+server :server, "localhost" do |server|
+  server.load File.expand_path(File.join(File.dirname(__FILE__), 'server.rb'))
+end
+
+node :node_odd do |node|
+  node.load File.join(current_dir, 'sum.rb')
+  node.group :odd
+end
+
+node :node_even do |node|
+  node.load File.join(current_dir, 'sum.rb')
+  node.group :even
+end

data/example/group/server.rb

@@ -0,0 +1,27 @@
+# 
+# Usage:
+#  drbqs-server server.rb -- 30 50
+#  drbqs-server server.rb -- 100 500 --step 100
+# 
+
+require_relative 'sum.rb'
+
+DRbQS.option_parser do |opt, hash|
+  opt.on('--step NUM', Integer) do |v|
+    hash[:step] = v
+  end
+end
+
+DRbQS.define_server do |server, argv, opts|
+  start_num = (argv[0] || 10).to_i
+  end_num = (argv[1] || 100).to_i
+  step_num = opts[:step] || 10
+
+  server.task_generator(:generate => 10) do |tgen|
+    start_num.step(end_num, step_num).with_index do |i, count|
+      create_add(Sum.new(i - 10, i), :exec, group: (count.even? ? :even : :odd)) do |srv, ret|
+        puts "Receive: #{ret.inspect}"
+      end
+    end
+  end
+end

data/example/group/sum.rb

@@ -0,0 +1,9 @@
+class Sum
+  def initialize(start_num, end_num)
+    @num = [start_num, end_num]
+  end
+
+  def exec
+    (@num[0]..@num[1]).inject(0) { |sum, i| sum += i }
+  end
+end

data/example/mandelbrot/README.md

@@ -10,6 +10,14 @@ This program requires gunzip command.
 
     drbqs-server server.rb -h
 
+or
+
+    drbqs-execute execute.rb -h
+
 ## Execute
 
     drbqs-server server.rb --execute-node <process_number>
+
+or
+
+    bin/drbqs-execute example/mandelbrot/execute.rb

data/example/mandelbrot/execute.rb

@@ -1,5 +1,9 @@
 DIR = File.dirname(__FILE__)
 
+current_dir = File.expand_path(File.dirname(__FILE__))
+
+usage :message => "Calculate Mandelbrot set", :server => File.join(current_dir, 'server.rb')
+
 server :local_server, "localhost" do |srv|
   srv.load File.join(DIR, 'server.rb')
 end

data/lib/drbqs/command_line/command_node.rb

@@ -15,6 +15,7 @@ HELP
         argv = option_parser_base(argv, :log_level => true, :daemon => true, :debug => true) do
           set(:load, '-l', '--load FILE', String, 'Add a file to load.')
           set(:process, '-P', '--process NUM', Integer, 'Set the number of node processes to execute.')
+          set(:group, '--group STR', String, 'Set the group of node.')
           set(:loadavg, '--loadavg STR', String, 'Set the threshold load average to sleep.')
           set(:log_prefix, '--log-prefix STR', String, "Set the prefix of log files. The default is '#{setting.default[:log_prefix][0]}'.")
           set(:log_stdout, '--log-stdout', 'Use stdout for outputting logs. This option cancels --log-prefix.')

data/lib/drbqs/execute/execute_node.rb

@@ -1,8 +1,6 @@
 module DRbQS
   class Execution
     class ExecuteNode
-      attr_reader :pid
-
       def initialize(uri, log_prefix, log_level, node_opts = {})
         @uri = uri
         @log_level = log_level
@@ -13,7 +11,6 @@ module DRbQS
         else
           @fname = nil
         end
-        @pid = []
         @node_opts = node_opts
       end
 
@@ -25,25 +22,11 @@ module DRbQS
       end
       private :get_log_file
 
-      def create_process
-        @pid << fork do
-          opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
-          node = DRbQS::Node.new(@uri, opts)
-          node.connect
-          node.calculate
-        end
-      end
-      private :create_process
-
       def execute(process_num, interval = 0)
-        process_num.times do |i|
-          create_process
-          sleep(interval) if interval > 0
-        end
-      end
-
-      def wait
-        Process.waitall
+        opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file, :process => process_num })
+        node = DRbQS::Node.new(@uri, opts)
+        node.connect
+        node.calculate
       end
     end
   end

data/lib/drbqs/node/connection.rb

@@ -28,8 +28,7 @@ module DRbQS
 
       def get_special_task(label)
         begin
-          ary = @message.read([label, nil, Symbol, nil], 0)
-          ary[1..-1]
+          @message.read([label, Array], 0)[1]
         rescue Rinda::RequestExpiredError
           nil
         end

data/lib/drbqs/node/node.rb

@@ -4,6 +4,7 @@ require 'drbqs/utility/transfer/transfer_client'
 require 'drbqs/node/connection'
 require 'drbqs/node/task_client'
 require 'drbqs/node/state'
+require 'drbqs/worker/worker'
 
 module DRbQS
 
@@ -13,25 +14,41 @@ module DRbQS
     PRIORITY_CALCULATE = 0
     OUTPUT_NOT_SEND_RESULT = 'not_send_result'
     DEFAULT_LOG_FILE = 'drbqs_client.log'
-    INTERVAL_TIME_DEFAULT = 1
+    INTERVAL_TIME_DEFAULT = 0.1
+    SAME_HOST_GROUP = :local
 
-    # :continue
-    # :max_loadavg
-    # :sleep_time
+    # @param [String] acces_uri Set the uri of server
+    # @param [Hash] opts Options of a node
+    # @option opts [Fixnum] :process Number of worker processes
+    # @option opts [Array] :group An array of group symbols
+    # @option opts [Boolean] :continue If we set true then the node process does not exit
+    # @option opts [Fixnum] :sleep_time Time interval during sleep of the node
+    # @option opts [String] :max_loadavg Note that this optiono is experimental
     def initialize(access_uri, opts = {})
       @access_uri = access_uri
       @logger = DRbQS::Misc.create_logger(opts[:log_file] || DEFAULT_LOG_FILE, opts[:log_level])
       @connection = nil
       @task_client = nil
-      @state = DRbQS::Node::State.new(:wait, :max_loadavg => opts[:max_loadavg], :sleep_time => opts[:sleep_time])
+      @worker_number = opts[:process] || 1
+      @state = DRbQS::Node::State.new(:wait, @worker_number, :max_loadavg => opts[:max_loadavg], :sleep_time => opts[:sleep_time])
       @process_continue = opts[:continue]
-      @signal_queue = Queue.new
+      @group = opts[:group] || []
+      @signal_to_server_queue = Queue.new
       @config = DRbQS::Config.new
+      @special_task_number = 0
+      @worker = DRbQS::Worker::ProcessSet.new(DRbQS::Worker::ForkedProcess)
+      @worker.on_result do |proc_key, res|
+        task_id, h = res
+        queue_result(task_id, h)
+      end
+      @worker.on_error do |proc_key, res|
+        @signal_to_server_queue.push([:node_error, res])
+      end
     end
 
-    def transfer_file
+    def transfer_file(files)
       begin
-        DRbQS::Transfer::Client.transfer_to_server
+        DRbQS::Transfer::Client.transfer_to_server(files)
       rescue Exception => err
         @logger.error("Fail to transfer files.") do
           "#{err.to_s} (#{err.class})\n#{err.backtrace.join("\n")}"
@@ -41,33 +58,62 @@ module DRbQS
     end
     private :transfer_file
 
-    def execute_task(marshal_obj, method_sym, args)
-      result = DRbQS::Task.execute_task(marshal_obj, method_sym, args)
-      transfer_file
-      DRbQS::Temporary.delete
-      result
+    def queue_result(task_id, result_hash)
+      if files = result_hash[:transfer]
+        transfer_file(files)
+      end
+      if subdir = result_hash[:tmp]
+        FileUtils.rm_r(result_hash[:tmp])
+      end
+      @task_client.queue_result(task_id, result_hash[:result])
     end
-    private :execute_task
+    private :queue_result
 
     def node_data
       { :uri => @access_uri }
     end
     private :node_data
 
+    # @param [Array] task_ary An array from @connection.get_initialization or @connection.get_finalization.
+    def send_special_task_ary_to_all_workers(task_ary)
+      task_ary.each do |ary|
+        ary_to_send = [nil] + ary
+        @state.each_worker_id do |wid|
+          @worker.send_task(wid, ary_to_send)
+        end
+      end
+    end
+    private :send_special_task_ary_to_all_workers
+
+    # Connect to the server and finish initialization of the node.
     def connect
       obj = DRbObject.new_with_uri(@access_uri)
       @server_key = obj[:key]
-      @connection = Node::Connection.new(obj[:message], @logger)
-      @task_client = Node::TaskClient.new(@connection.node_number, obj[:queue], obj[:result], @logger)
+      @connection = DRbQS::Node::Connection.new(obj[:message], @logger)
+      set_node_group_for_task
+      @task_client = DRbQS::Node::TaskClient.new(@connection.node_number, obj[:queue], obj[:result],
+                                                 @group, @logger)
       DRbQS::Transfer::Client.set(obj[:transfer].get_client(server_on_same_host?)) if obj[:transfer]
-      if ary = @connection.get_initialization
-        execute_task(*ary)
+      @state.each_worker_id do |wid|
+        @worker.create_process(wid)
+      end
+      if ary_initialization = @connection.get_initialization
+        send_special_task_ary_to_all_workers(ary_initialization)
       end
       @config.list.node.save(Process.pid, node_data)
     end
 
+    # This method must be executed after @connection is set.
+    def set_node_group_for_task
+      if server_on_same_host?
+        @group << DRbQS::Node::SAME_HOST_GROUP
+      end
+    end
+    private :set_node_group_for_task
+
     def server_on_same_host?
-      @config.list.server.server_of_key_exist?(@access_uri, @server_key)
+      @server_on_same_host ||
+        (@server_on_same_host = @config.list.server.server_of_key_exist?(@access_uri, @server_key))
     end
 
     def dump_not_send_result_to_file
@@ -83,17 +129,10 @@ module DRbQS
     end
     private :output_error
 
-    def process_exit
-      dump_not_send_result_to_file
-      unless @process_continue
-        Kernel.exit
-      end
-    end
-    private :process_exit
-
     def execute_finalization
-      if ary = @connection.get_finalization
-        execute_task(*ary)
+      @worker.waitall
+      if ary_finalization = @connection.get_finalization
+        send_special_task_ary_to_all_workers(ary_finalization)
       end
     rescue => err
       output_error(err, "On finalization")
@@ -102,80 +141,88 @@ module DRbQS
 
     def send_error(err, mes)
       output_error(err, mes)
-      @connection.send_node_error("#{err.to_s}\n#{err.backtrace.join("\n")}")
+      begin
+        @connection.send_node_error("#{err.to_s}\n#{err.backtrace.join("\n")}")
+      rescue
+      end
     end
     private :send_error
 
     def get_new_task
-      if @state.request?
-        if @state.change_to_sleep_for_busy_system
-          @logger.info("Sleep because system is busy.")
-        elsif @task_client.add_new_task
-          @state.change_to_calculate
-        end
+      if @state.request? && (obtained_task_id = @task_client.add_new_task(@state.request_task_number))
+        return obtained_task_id
       end
+      nil
     end
     private :get_new_task
 
-    def send_result
-      flag_finilize_exit = @task_client.send_result
-      if @state.calculate? && !@task_client.calculating_task
-        @state.change_to_finish_calculating
+    def send_result_to_server
+      if sent_task_id = @task_client.send_result
+        @state.set_finish_of_task(sent_task_id)
       end
-      flag_finilize_exit
     end
-    private :send_result
+    private :send_result_to_server
 
+    # Send signals from @signal_to_server_queue,
+    # which stores errors of workers and signals to current process.
+    # @return [Boolean] If some error signal is sent then this method returns true. Otherwise, nil.
     def send_signal
-      until @signal_queue.empty?
-        signal, obj = @signal_queue.pop
+      flag_finalize_exit = nil
+      until @signal_to_server_queue.empty?
+        signal, obj = @signal_to_server_queue.pop
         case signal
         when :node_error
           send_error(obj, "Communicating with server")
-          process_exit
+          dump_not_send_result_to_file
+          flag_finalize_exit = true
+        when :signal_kill
+          flag_finalize_exit = true
+        else
+          raise "Not implemented"
         end
       end
+      flag_finalize_exit
     end
     private :send_signal
 
-    def process_signal
+    # If the method returns true, the node finishes.
+    def process_signal_for_server
+      flag_finalize_exit = send_signal
       case @connection.respond_signal
       when :wake
-        @state.change_to_wait
+        @state.wakeup_sleeping_worker
       when :sleep
         @state.change_to_sleep
       when :exit
-        return :exit
+        return nil
       when :finalize
-        return :finalize
+        flag_finalize_exit = true
       when :exit_after_task
-        @task_client.set_exit_after_task
+        @state.set_exit_after_task
         @process_continue = nil
       end
-      nil
-    end
-    private :process_signal
-
-    def communicate_with_server
-      get_new_task
-      sig = process_signal
-      return nil if sig == :exit
-      flag_finilize_exit = send_result
-      send_signal
-      if sig == :finalize || flag_finilize_exit
+      if flag_finalize_exit
         execute_finalization
         return nil
       end
       @state.wakeup_automatically_for_unbusy_system
       true
     end
-    private :communicate_with_server
+    private :process_signal_for_server
 
-    def calculate_task
-      marshal_obj, method_sym, args = @task_client.dequeue_task
-      @task_client.queue_result(execute_task(marshal_obj, method_sym, args))
+    # Dequeue tasks from @task_client and send them to worker processes.
+    def send_task_to_worker
+      wids = @state.waiting_worker_id
+      wids.each do |wid|
+        if ary = @task_client.dequeue_task
+          @state.set_calculating_task(wid, ary[0])
+          @worker.send_task(wid, ary)
+        else
+          break
+        end
+      end
     end
-    private :calculate_task
+    private :send_task_to_worker
 
     def clear_node_files
       DRbQS::Temporary.delete_all
@@ -188,52 +235,66 @@ module DRbQS
     end
     private :wait_interval_of_connection
 
-    def thread_communicate
-      Thread.new do
-        begin
-          loop do
-            unless communicate_with_server
-              clear_node_files
-              break
-            end
-            wait_interval_of_connection
-          end
-        rescue => err
-          send_error(err, "Calculating thread")
-        ensure
-          process_exit
-        end
+    def set_signal_trap
+      Signal.trap(:TERM) do
+        @signal_to_server_queue.push([:signal_kill])
       end
     end
-    private :thread_communicate
 
-    def thread_calculate
-      Thread.new do
-        begin
-          loop do
-            calculate_task
-          end
-        rescue => err
-          @signal_queue.push([:node_error, err])
-        end
-      end
+    MAX_WAIT_FINISH = 3
+    WAIT_INTERVAL = 0.1
+
+    def respond_worker_signal
+      @worker.respond_signal
     end
-    private :thread_calculate
+    private :respond_worker_signal
 
-    def set_signal_trap
-      Signal.trap(:TERM) do
-        process_exit
+    def wait_process_finish
+      @worker.prepare_to_exit
+      total_wait_time = 0.0
+      loop do
+        respond_worker_signal
+        if !@worker.has_process?
+          break
+        elsif total_wait_time > MAX_WAIT_FINISH
+          # Kill worker processes forcibly.
+          @worker.kill_all_processes
+          break
+        end
+        sleep(WAIT_INTERVAL)
+        total_wait_time += WAIT_INTERVAL
       end
+      send_result_to_server
     end
+    private :wait_process_finish
 
     def calculate(opts = {})
       set_signal_trap
-      cn = thread_communicate
-      exec = thread_calculate
-      cn.priority = PRIORITY_RESPOND
-      exec.priority = PRIORITY_CALCULATE
-      cn.join
+      begin
+        loop do
+          send_result_to_server
+          unless process_signal_for_server
+            break
+          end
+          if @state.change_to_sleep_for_busy_system
+            @logger.info("Sleep because system is busy.")
+          end
+          if get_new_task
+            send_task_to_worker
+          elsif @state.all_workers_waiting? && !@process_continue
+            execute_finalization
+            break
+          end
+          unless respond_worker_signal
+            wait_interval_of_connection
+          end
+        end
+      rescue => err
+        send_error(err, "Node error occurs.")
+        @worker.kill_all_processes
+      end
+      wait_process_finish
+      clear_node_files
     end
   end
-
 end