drbqs 0.0.14 → 0.0.15

data/lib/drbqs/manage/ssh_shell.rb

@@ -6,39 +6,81 @@ module DRbQS
   class Manage
     # Requirements:
     #   bash
-    #   nohup
     class SSHShell
+      class RubyEnvironment
+        DEFAULT_RVM_SCRIPT = '$HOME/.rvm/scripts/rvm'
+
+        attr_reader :directory, :rvm, :rvm_init, :env
+
+        # :directory
+        # :rvm
+        # :rvm_init
+        # :env
+        def initialize(opts = {})
+          @directory = opts[:directory]
+          @rvm = opts[:rvm]
+          @rvm_init = opts[:rvm_init]
+          if (@rvm || @rvm_init) && !(String === @rvm_init)
+            @rvm_init = DEFAULT_RVM_SCRIPT
+          end
+          @env = opts[:env]
+        end
+
+        def commands_to_set_env_on_bash
+          if @env
+            @env.map do |var, val|
+              "export #{var}=#{val}"
+            end
+          else
+            []
+          end
+        end
+        private :commands_to_set_env_on_bash
+
+        def setup_commands
+          cmds = commands_to_set_env_on_bash
+          cmds << "cd #{@directory}" if @directory
+          cmds << "source #{@rvm_init}" if @rvm_init
+          cmds << "rvm use #{@rvm}" if @rvm
+          cmds
+        end
+
+        def get_environment_commands
+          ['echo "directory: " `pwd`',
+           'echo "files:"',
+           'ls',
+           'if which rvm > /dev/null; then rvm info; else ruby -v; fi']
+        end
+      end
+
       class InvalidDestination < StandardError
       end
       class GetInvalidExitStatus < StandardError
       end
 
-      attr_reader :user, :host, :directory, :port
-
-      DEFAULT_RVM_SCRIPT = '$HOME/.rvm/scripts/rvm'
-      DEFAULT_OUTPUT_FILE = 'drbqs_nohup.log'
+      attr_reader :user, :host, :port, :keys
 
       # :shell     shell to use
+      # :env       a hash of environmental variables and their values
       # :dir       base directory of ssh server
       # :rvm       version of ruby on rvm
       # :rvm_init  path of script to initialize rvm
-      # :output    file to output of stdout and stderr
+      # :keys      path of a ssh key
+      # :io        IO to output results of commands
       def initialize(dest, opts = {})
         @user, @host, @port = split_destination(dest)
         if !(@host && @user)
           raise InvalidDestination, "Invalid destination of ssh server."
         end
+        @keys = opts.delete(:keys)
         @shell = opts[:shell] || 'bash'
-        @rvm = opts[:rvm]
-        @rvm_init = opts[:rvm_init]
-        if (@rvm || @rvm_init) && !(String === @rvm_init)
-          @rvm_init = DEFAULT_RVM_SCRIPT
-        end
-        @nohup_output = opts[:output] || DEFAULT_OUTPUT_FILE
-        @directory = opts[:dir]
-        @out = $stdout
-        @nice = opts[:nice]
-        @nohup = opts[:nohup]
+        @ruby_environment = DRbQS::Manage::SSHShell::RubyEnvironment.new(opts)
+        @out = opts[:io]
+        @ssh = nil
+      end
+
+      def directory
+        @ruby_environment.directory
       end
 
       def split_destination(dest)
@@ -62,11 +104,14 @@ module DRbQS
       private :split_destination
 
       def output_command(cmd, result)
-        @out.puts "#{@user}@#{@host}$ #{cmd}" if @out
-        @out.print result
+        if @out
+          @out.puts "#{@user}@#{@host}$ #{cmd}"
+          @out.print result
+        end
       end
       private :output_command
 
+      # Return an array of a Net::SSH::Shell::Process object and a result string.
       def shell_exec_get_output(sh, cmd)
         result = ''
         pr_cmd = sh.execute!(cmd) do |sh_proc|
@@ -90,53 +135,53 @@ module DRbQS
 
       def shell_exec_check(sh, cmd)
         ary = shell_exec(sh, cmd)
-        if ary[0].exit_status != 0
-          raise GetInvalidExitStatus, "Can not execute '#{cmd}' on #{@host} properly."
+        n = ary[0].exit_status
+        if n != 0
+          raise GetInvalidExitStatus, "Can not execute properly on #{@host}.\nExit status: #{n}\ncommand: #{cmd}"
         end
         ary
       end
       private :shell_exec_check
 
-      def execute_command(&block)
-        Net::SSH.start(@host, @user, :port => @port) do |ssh|
+      def exec(cmd, opts = {})
+        unless @ssh
+          raise "Not connect."
+        end
+        if opts[:check]
+          shell_exec_check(@ssh, cmd)
+        else
+          shell_exec(@ssh, cmd)
+        end
+      end
+
+      def start(&block)
+        Net::SSH.start(@host, @user, :port => @port, :keys => @keys) do |ssh|
           ssh.shell(@shell) do |sh|
-            shell_exec_check(sh, "cd #{@directory}") if @directory
-            shell_exec_check(sh, "source #{@rvm_init}") if @rvm_init
-            shell_exec_check(sh, "rvm use #{@rvm}") if @rvm
-            yield(sh)
+            @ruby_environment.setup_commands.each do |cmd|
+              shell_exec_check(sh, cmd)
+            end
+            @ssh = sh
+            yield(self)
             shell_exec(sh, "exit")
           end
         end
+      ensure
+        @ssh = nil
       end
-      private :execute_command
 
-      def get_environment
-        execute_command do |sh|
-          ['echo "directory: " `pwd`',
-           'echo "files:"',
-           'ls',
-           'if which rvm > /dev/null; then rvm info; else ruby -v; fi'].each do |cmd|
-            shell_exec(sh, cmd)
+      # :check
+      def execute_all(commands, opts = {})
+        results = []
+        start do |ssh_shell|
+          commands.each do |cmd|
+            results << ssh_shell.exec(cmd, opts)
           end
         end
+        results
       end
 
-      def start(*args)
-        cmd = args.join(' ')
-        if @nice
-          if Integer === @nice
-            cmd = "nice -n #{@nice.to_s} " + cmd
-          else
-            cmd = "nice " + cmd
-          end
-        end
-        execute_command do |sh|
-          if @nohup
-            pr, path = shell_exec_check(sh, "filename-create new -p middle -D parent -t time #{@nohup_output}")
-            cmd = "nohup #{cmd} > #{path.strip} 2>&1 &"
-          end
-          shell_exec(sh, cmd)
-        end
+      def get_environment
+        execute_all(@ruby_environment.get_environment_commands)
       end
     end
   end

data/lib/drbqs/node/connection.rb

@@ -52,7 +52,7 @@ module DRbQS
           when :alive_p
             @message.write([:server, :alive, @node_number])
             @logger.info("Send alive signal of node id #{@node_number}")
-          when :exit, :finalize, :exit_after_task
+          when :exit, :finalize, :exit_after_task, :sleep, :wake
             return sym
           else
             raise "Get invalid signal: #{sym.inspect}"

data/lib/drbqs/node/node.rb

@@ -1,25 +1,29 @@
 require 'drbqs/task/task'
 require 'drbqs/utility/transfer/transfer_client'
+require 'drbqs/utility/temporary'
 require 'drbqs/node/connection'
 require 'drbqs/node/task_client'
-require 'drbqs/utility/temporary'
+require 'drbqs/node/state'
 
 module DRbQS
 
   class Node
 
-    WAIT_NEW_TASK = 1
     PRIORITY_RESPOND = 10
     PRIORITY_CALCULATE = 0
     OUTPUT_NOT_SEND_RESULT = 'not_send_result'
     DEFAULT_LOG_FILE = 'drbqs_client.log'
+    INTERVAL_TIME_DEFAULT = 1
 
     # :continue
+    # :max_loadavg
+    # :sleep_time
     def initialize(access_uri, opts = {})
       @access_uri = access_uri
       @logger = DRbQS::Misc.create_logger(opts[:log_file] || DEFAULT_LOG_FILE, opts[:log_level])
       @connection = nil
       @task_client = nil
+      @state = DRbQS::Node::State.new(:wait, :max_loadavg => opts[:max_loadavg], :sleep_time => opts[:sleep_time])
       @process_continue = opts[:continue]
       @signal_queue = Queue.new
       @config = DRbQS::Config.new
@@ -108,19 +112,27 @@ module DRbQS
     end
     private :send_error
 
-    def communicate_with_server
-      flag_finilize_exit = false
-      @task_client.add_new_task
-      case @connection.respond_signal
-      when :exit
-        return nil
-      when :finalize
-        flag_finilize_exit = true
-      when :exit_after_task
-        @task_client.set_exit_after_task
-        @process_continue = nil
+    def get_new_task
+      if @state.request?
+        if @state.change_to_sleep_for_busy_system
+          @logger.info("Sleep because system is busy.")
+        elsif @task_client.add_new_task
+          @state.change_to_calculate
+        end
       end
+    end
+    private :get_new_task
+
+    def send_result
       flag_finilize_exit = @task_client.send_result
+      if @state.calculate? && !@task_client.calculating_task
+        @state.change_to_finish_calculating
+      end
+      flag_finilize_exit
+    end
+    private :send_result
+
+    def send_signal
       until @signal_queue.empty?
         signal, obj = @signal_queue.pop
         case signal
@@ -129,10 +141,38 @@ module DRbQS
           process_exit
         end
       end
-      if flag_finilize_exit
+    end
+    private :send_signal
+
+    def process_signal
+      case @connection.respond_signal
+      when :wake
+        @state.change_to_wait
+      when :sleep
+        @state.change_to_sleep
+      when :exit
+        return :exit
+      when :finalize
+        return :finalize
+      when :exit_after_task
+        @task_client.set_exit_after_task
+        @process_continue = nil
+      end
+      nil
+    end
+    private :process_signal
+
+    def communicate_with_server
+      get_new_task
+      sig = process_signal
+      return nil if sig == :exit
+      flag_finilize_exit = send_result
+      send_signal
+      if sig == :finalize || flag_finilize_exit
         execute_finalization
         return nil
       end
+      @state.wakeup_automatically_for_unbusy_system
       true
     end
     private :communicate_with_server
@@ -143,16 +183,26 @@ module DRbQS
     end
     private :calculate_task
 
+    def clear_node_files
+      DRbQS::Temporary.delete_all
+      @config.list.node.delete(Process.pid)
+    end
+    private :clear_node_files
+
+    def wait_interval_of_connection
+      sleep(INTERVAL_TIME_DEFAULT)
+    end
+    private :wait_interval_of_connection
+
     def thread_communicate
       Thread.new do
         begin
           loop do
             unless communicate_with_server
-              DRbQS::Temporary.delete_all
-              @config.list.node.delete(Process.pid)
+              clear_node_files
               break
             end
-            sleep(WAIT_NEW_TASK)
+            wait_interval_of_connection
           end
         rescue => err
           send_error(err, "Calculating thread")

data/lib/drbqs/node/state.rb

@@ -0,0 +1,109 @@
+module DRbQS
+  class Node
+    # Value of state is :sleep, :wait, or :calculate.
+    class State
+      attr_reader :state
+
+      ALL_STATES = [:sleep, :wait, :calculate]
+      DEFAULT_SLEEP_TIME = 300
+      LOADAVG_PATH = '/proc/loadavg'
+
+      def initialize(state, opts = {})
+        @state = state
+        @sleep_at_calculated = nil
+        @load_average_threshold = opts[:max_loadavg]
+        @sleep_time = opts[:sleep_time] || DEFAULT_SLEEP_TIME
+        @auto_wakeup = nil
+      end
+
+      def change(state)
+        unless ALL_STATES.include?(state)
+          raise ArgumentError, "Invalid state of node '#{state}'."
+        end
+        @state = state
+      end
+
+      def calculate?
+        @state == :calculate
+      end
+
+      def rest?
+        !calculate?
+      end
+
+      def stop?
+        @state == :sleep
+      end
+
+      def request?
+        @state == :wait
+      end
+
+      def change_to_wait
+        unless calculate?
+          change(:wait)
+        end
+      end
+
+      def change_to_sleep
+        if calculate?
+          @sleep_at_calculated = true
+        else
+          change(:sleep)
+        end
+      end
+
+      def change_to_calculate
+        change(:calculate)
+      end
+
+      def change_to_finish_calculating
+        if @sleep_at_calculated
+          change(:sleep)
+        else
+          change(:wait)
+        end
+        @sleep_at_calculated = nil
+      end
+
+      def sleep_with_auto_wakeup
+        change(:sleep)
+        @auto_wakeup = Time.now + @sleep_time
+      end
+
+      def wakeup_automatically_for_unbusy_system
+        if @auto_wakeup && Time.now > @auto_wakeup && !system_busy?
+          change(:wait)
+          @auto_wakeup = nil
+          return true
+        end
+        nil
+      end
+
+      def get_load_average
+        File.read(LOADAVG_PATH).split[0..2].map do |s|
+          s.to_f
+        end
+      end
+      private :get_load_average
+
+      def system_busy?
+        if @load_average_threshold && File.exist?(LOADAVG_PATH)
+          avg = get_load_average
+          if (avg[0] + avg[1]) / 2 >= @load_average_threshold
+            return true
+          end
+        end
+        nil
+      end
+
+      def change_to_sleep_for_busy_system
+        if system_busy?
+          sleep_with_auto_wakeup
+          return true
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/drbqs/node/task_client.rb

@@ -53,14 +53,14 @@ module DRbQS
       end
 
       def add_new_task
-        if !@calculating_task && !@exit_after_task
-          if ary = get_task
-            task_id, obj, method_sym, args = ary
-            @logger.info("Send accept signal: node #{@node_number} caluclating #{task_id}")
-            @result.write([:accept, task_id, @node_number])
-            queue_task(task_id, [obj, method_sym, args])
-          end
+        if !@calculating_task && !@exit_after_task && (ary = get_task)
+          task_id, obj, method_sym, args = ary
+          @logger.info("Send accept signal: node #{@node_number} caluclating #{task_id}")
+          @result.write([:accept, task_id, @node_number])
+          queue_task(task_id, [obj, method_sym, args])
+          return true
         end
+        nil
       end
 
       # If the method return true, a node should finilize and exit.

data/lib/drbqs/server/history.rb

@@ -28,6 +28,22 @@ module DRbQS
       def each(&block)
         @data.each(&block)
       end
+    end
+
+    class TaskHistory < DRbQS::Server::History
+      attr_reader :finished_task_number
+
+      def initialize
+        super
+        @finished_task_number = 0
+      end
+
+      def set(id, *args)
+        if args[0] == :result
+          @finished_task_number += 1
+        end
+        super(id, *args)
+      end
 
       def log_strings
         s = ''

data/lib/drbqs/server/message.rb

@@ -11,6 +11,14 @@ module DRbQS
         @logger = logger
       end
 
+      # Returned values:
+      # [:exit_server, nil]
+      # [:request_status, nil]
+      # [:request_history, nil]
+      # [:exit_after_task, node_id]
+      # [:wake_node, node_id]
+      # [:sleep_node, node_id]
+      # [:node_error, [node_id, error_message]]
       def get_message
         begin
           mes = @message.take([:server, Symbol, nil], 0)
@@ -36,10 +44,20 @@ module DRbQS
           return [mes, arg]
         when :request_status
           @logger.info("Get status request from #{arg.to_s}")
+        when :request_history
+          @logger.info("Get history request from #{arg.to_s}")
+        when :sleep_node
+          @logger.info("Get sleep node message for node #{arg.to_s}")
+          return [mes, arg]
+        when :wake_node
+          @logger.info("Get wake node message for node #{arg.to_s}")
+          return [mes, arg]
         when :node_error
-          @node_list.delete(arg[0])
+          @node_list.delete(arg[0], :error)
           @logger.info("Node Error (#{arg[0]})") { arg[1] }
           return [mes, arg[0]]
+        when :new_data
+          return [mes, arg]
         else
           @logger.error("Invalid message from #{arg.to_s}")
           return nil
@@ -59,7 +77,9 @@ module DRbQS
       end
 
       def send_signal(node_id, signal)
-        @message.write([node_id, signal])
+        if node_exist?(node_id)
+          @message.write([node_id, signal])
+        end
       end
       private :send_signal
 
@@ -75,30 +95,56 @@ module DRbQS
         send_signal_to_all_nodes(:exit)
       end
 
+      def send_sleep(node_id)
+        send_signal(node_id, :sleep)
+      end
+
+      def send_wake(node_id)
+        send_signal(node_id, :wake)
+      end
+
       # Send all nodes a message to finalize and exit.
       def send_finalization
         send_signal_to_all_nodes(:finalize)
       end
 
       def send_exit_after_task(node_id)
+        @node_list.add_to_preparation_to_exit(node_id)
         send_signal(node_id, :exit_after_task)
       end
 
-      def send_status(calculating_task_id)
-        s = ''
-        @node_list.history.each do |node_id, events|
-          if events.size == 0 || events.size > 2
-            raise "Invalid history of nodes: #{events.inspect}"
-          end
-          connect = events[0]
-          s << sprintf("%4d %s\t", node_id, connect[2])
-          if disconnect = events[1]
-            s << "disconnected: (#{time_to_history_string(connect[0])} - #{time_to_history_string(disconnect[0])})\n"
-          else
-            task_ids = calculating_task_id[node_id].to_a
-            s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_history_string(connect[0])})\n"
+      # +data+ is a hash including server information.
+      # The keys are :calculating_task_number, :finished_task_number, :stocked_task_number,
+      # :calculating_nodes, and :generator_number.
+      def send_status(data)
+        s = "Nodes:\n"
+        if @node_list.history.size == 0
+          s << "  none\n"
+        else
+          @node_list.history.each do |node_id, events|
+            if events.size == 0
+              s << "Empty history of node #{node_id}\n"
+            else
+              connect = events[0]
+              s << sprintf("%4d %s\t", node_id, connect[2])
+              if events.size > 1
+                s << "start:#{time_to_history_string(connect[0])}"
+                events[1..-1].each do |t, key|
+                  s << ", #{key}: #{time_to_history_string(t)}"
+                end
+                s << "\n"
+              elsif data[:calculating_nodes]
+                task_ids = data[:calculating_nodes][node_id].to_a
+                s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_history_string(connect[0])})\n"
+              end
+            end
           end
         end
+        s << "Server:\n"
+        s << "  calculating tasks: #{data[:calculating_task_number]}\n"
+        s << "  finished tasks   : #{data[:finished_task_number]}\n"
+        s << "  stocked tasks    : #{data[:stocked_task_number]}\n"
+        s << "  task generator   : #{data[:generator_number]}"
         begin
           @message.take([:status, nil], 0)
         rescue Rinda::RequestExpiredError
@@ -106,6 +152,14 @@ module DRbQS
         @message.write([:status, s])
       end
 
+      def send_history(history_string)
+        begin
+          @message.take([:history, nil], 0)
+        rescue Rinda::RequestExpiredError
+        end
+        @message.write([:history, history_string])
+      end
+
       def get_all_nodes
         @node_list.list.dup
       end
@@ -136,6 +190,17 @@ module DRbQS
       def set_finalization(task)
         set_special_task(:finalization, task)
       end
+
+      def shutdown_unused_nodes(calculating_nodes)
+        shutdown_nodes = []
+        @node_list.each do |node_id, id_str|
+          if !@node_list.prepare_to_exit?(node_id) && !calculating_nodes.include?(node_id)
+            send_exit_after_task(node_id)
+            shutdown_nodes << node_id
+          end
+        end
+        shutdown_nodes
+      end
     end
 
   end