drbqs 0.0.14 → 0.0.15

data/lib/drbqs/execute/process_define.rb

@@ -0,0 +1,213 @@
+require 'drbqs/execute/register'
+
+module DRbQS
+  class ProcessDefinition
+    class InvalidServerDefinition < StandardError
+    end
+
+    class InvalidNodeDefinition < StandardError
+    end
+
+    attr_reader :register
+
+    def initialize(server, node, port, io = nil)
+      @server = server
+      @node = node
+      @port = port
+      @register = DRbQS::ProcessDefinition::Register.new
+      @io = io
+    end
+
+    def puts_progress(str)
+      @io.puts str if @io
+    end
+    private :puts_progress
+
+    # Log directory for processes on localhost.
+    # Processes over ssh does not use this directory.
+    def local_log_directory
+      @logal_log_directory ||= FileName.create(default_value(:log) || 'drbqs_execute_log')
+    end
+
+    def load(path)
+      @register.__load__(path)
+    end
+
+    def get_server_setting(name = nil)
+      if data = (name ? @register.__server__.assoc(name.intern) : @register.__server__[0])
+        { :name => data[0], :type => data[1][:type], :setting => data[1][:setting], :hostname => data[1][:args][0] }
+      elsif name
+        get_server_setting(nil)
+      else
+        nil
+      end
+    end
+    private :get_server_setting
+
+    def get_node_data(name)
+      if ary = @register.__node__.assoc(name)
+        ary[1]
+      else
+        nil
+      end
+    end
+    private :get_node_data
+
+    def each_node(names = nil, &block)
+      if names
+        names.each do |name|
+          if data = get_node_data(name)
+            yield(name, data)
+          end
+        end
+      else
+        @register.__node__.each do |name, data|
+          yield(name, data)
+        end
+      end
+    end
+    private :each_node
+
+    def default_value(key)
+      @register.__default__[key]
+    end
+    private :default_value
+
+    def server_port
+      @port || default_value(:port) || ROOT_DEFAULT_PORT
+    end
+    private :server_port
+
+    def server_uri(name)
+      data = get_server_setting(name)
+      DRbQS::Misc.create_uri(:host => data[:hostname], :port => server_port)
+    end
+    private :server_uri
+
+    def execute_server(server_args)
+      if data = get_server_setting(@server)
+        name = data[:name].to_s
+        puts_progress "Execute server '#{name}' (#{data[:type]})"
+        setting = data[:setting]
+        hostname = data[:hostname]
+        type = data[:type]
+        if type == :ssh
+          setting.value.connect name
+          server_setting = setting.mode_setting
+        else
+          server_setting = setting
+          server_setting.value.daemon FileName.create(local_log_directory, "server_execute.log", :position => :middle)
+        end
+        server_setting.set_server_argument(*server_args)
+        server_setting.value.port server_port
+        unless server_setting.set?(:sftp_host)
+          server_setting.value.sftp_host hostname
+        end
+        setting.parse!
+        unless type == :ssh
+          server_setting.value.argument.each do |path|
+            unless File.exist?(path)
+              raise "File '#{path}' does not exist."
+            end
+          end
+        end
+        setting.exec
+      end
+    rescue Exception => err
+      puts_progress "Fail to execute server '#{data[:name].to_s}'"
+      mes = "#{err.to_s} (#{err.class.to_s})"
+      mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
+      new_err = InvalidServerDefinition.new(mes)
+      new_err.set_backtrace(err.backtrace)
+      raise new_err
+    end
+
+    def execute_one_node(name, data, uri)
+      puts_progress "Execute node '#{name}' (#{data[:type]})"
+      setting = data[:setting]
+      node_setting = (data[:type] == :ssh ? setting.mode_setting : setting)
+      node_setting.value.argument.clear
+      node_setting.value.connect uri
+      if data[:type] == :ssh
+        unless setting.set?(:connect)
+          setting.value.connect name.to_s
+        end
+      else
+        node_log_dir = FileName.create(local_log_directory, 'node_execute_log', :directory => :self)
+        setting.clear :log_stdout
+        setting.value.log_prefix File.join(node_log_dir, 'node')
+        setting.value.daemon File.join(node_log_dir, 'execute.log')
+      end
+      setting.parse!
+      setting.exec
+    rescue Exception => err
+      puts_progress "Fail to execute node '#{name.to_s}'"
+      mes = "#{err.to_s} (#{err.class.to_s})"
+      mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
+      new_err = InvalidNodeDefinition.new(mes)
+      new_err.set_backtrace(err.backtrace)
+      raise new_err
+    end
+    private :execute_one_node
+
+    def execute_node
+      uri = server_uri(@server)
+      each_node(@node) do |name, data|
+        execute_one_node(name, data, uri)
+      end
+    end
+
+    def information
+      info = {}
+      info[:server] = @register.__server__.map do |name, data|
+        [name, data[:type]]
+      end
+      info[:node] = @register.__node__.map do |name, data|
+        [name, data[:type]]
+      end
+      if data = get_server_setting(@server)
+        default_server = data[:name]
+      else
+        default_server = nil
+      end
+      info[:default] = {
+        :server => default_server,
+        :node => @node || info[:node].map { |ary| ary[0]},
+        :port => server_port
+      }
+      info
+    end
+
+    def information_string
+      info = information
+      str = "Server:\n"
+      info[:server].each do |name, type|
+        s = sprintf("%- 16s  %s\n", name, (type == :ssh ? 'ssh' : 'local'))
+        str << (info[:default][:server] == name ? "* " : "  ")
+        str << s
+      end
+      str << "Node:\n"
+      info[:node].each do |name, type|
+        s = sprintf("%- 16s  %s\n", name, (type == :ssh ? 'ssh' : 'local'))
+        str << (info[:default][:node].include?(name) ? "  " : "- ")
+        str << s
+      end
+      str << "Port: #{info[:default][:port]}"
+    end
+
+    def usage
+      if data = @register.__usage__
+        str = data[:message] ? "\nDescription:\n#{data[:message]}" : ""
+        if (server_file = data[:server]) && File.exist?(server_file)
+          Kernel.load(server_file)
+          if server_help = DRbQS.option_help_message
+            str << "\n\n" << server_help
+          end
+        end
+        str
+      else
+        ''
+      end
+    end
+  end
+end

data/lib/drbqs/execute/register.rb

@@ -0,0 +1,147 @@
+module DRbQS
+  class ProcessDefinition
+    class Register
+      attr_reader :__server__, :__node__, :__default__, :__usage__
+
+      def initialize
+        @__server__ = []
+        @__node__ = []
+        @__default__ = {}
+        @__usage__ = nil
+      end
+
+      def __register__(type, name, *args, &block)
+        if block_given?
+          type = type.intern
+          case type
+          when :server
+            setting = DRbQS::Setting::Server.new
+          when :node
+            setting = DRbQS::Setting::Node.new
+          else
+            raise ArgumentError, "Invalid type of setting '#{type.to_s}'"
+          end
+          case block.arity
+          when 2
+            ssh_setting = DRbQS::Setting::SSH.new
+            ssh_setting.value.argument << type.to_s
+            yield(setting.value, ssh_setting.value)
+            ssh_setting.mode_setting = setting
+            unless ssh_setting.value.argument[0] == type.to_s
+              ssh_setting.value.argument.unshift(type.to_s)
+            end
+            ary = [name.intern, { :type => :ssh, :setting => ssh_setting, :args => args }]
+          when 1
+            yield(setting.value)
+            ary = [name.intern, { :type => type, :setting => setting, :args => args }]
+          else
+            raise ArgumentError, "Block must take one or two arguments."
+          end
+          instance_variable_get("@__#{type.to_s}__") << ary
+        else
+          raise ArgumentError, "Block to define settings is not given."
+        end
+      end
+      private :__register__
+
+      # To set properties of server we can use the similar options to the command 'drbqs-server'.
+      # When we execute a server over ssh, we can use the similar options to the command 'drbqs-ssh'
+      # Exceptionally, we can set files to load by 'load' method
+      # and set a ssh server by 'connect' method.
+      # If we omit the 'connect' method then the program tries to connect
+      # the name specified as first argument.
+      # 
+      # * Example of a server on localhost
+      # register_server(:server_local, "example.com") do |server|
+      #   server.load "server_definition.rb"
+      #   server.acl "/path/to/acl"
+      #   server.log_file "/path/to/log"
+      #   server.log_level Logger::ERROR
+      #   server.sftp_user "username"
+      #   server.sftp_host "example.com"
+      # end
+      # 
+      # * Example of a server over ssh
+      # register_server(:server_ssh, "example.co.jp") do |server, ssh|
+      #   server.load "server_definition.rb"
+      #   server.acl "/path/to/acl"
+      #   server.log_level Logger::ERROR
+      #   server.sftp_user "username"
+      #   server.sftp_host "example.com"
+      # 
+      #   ssh.connect "hostname"
+      #   ssh.directory "/path/to/dir"
+      #   ssh.shell "bash"
+      #   ssh.rvm "ruby-head"
+      #   ssh.rvm_init "/path/to/scripts/rvm"
+      #   ssh.output "/path/to/output"
+      #   ssh.nice 10
+      # end
+      def register_server(name, hostname = nil, &block)
+        __register__(:server, name, hostname || name.to_s, &block)
+      end
+
+      # To set properties of nodes we can use the similar options to the command 'drbqs-node'.
+      # When we execute a server over ssh, we can use the similar options to the command 'drbqs-ssh'
+      # Exceptionally, we can set a ssh server by 'connect' method.
+      # If we omit the 'connect' method then the program tries to connect
+      # the name specified as first argument.
+      # 
+      # * Example of nodes on localhost
+      # register_node(:node_local) do |node|
+      #   node.process 3
+      #   node.load "load_lib.rb"
+      #   node.log_prefix "/path/to/log"
+      #   node.log_level Logger::DEBUG
+      # end
+      # 
+      # * Example of nodes over ssh
+      # register_node(:node_ssh) do |node, ssh|
+      #   node.process 3
+      #   node.load "load_lib.rb"
+      #   node.log_level Logger::DEBUG
+      # 
+      #   ssh.connect "hostname"
+      #   ssh.directory "/path/to/dir"
+      #   ssh.shell "bash"
+      #   ssh.rvm "ruby-head"
+      #   ssh.rvm_init "/path/to/scripts/rvm"
+      #   ssh.output "/path/to/output"
+      #   ssh.nice 10
+      # end
+      def register_node(name, &block)
+        __register__(:node, name, &block)
+      end
+
+      # We can set default server, default port, and default directory to output log.
+      # * Example of usage
+      #   default :port => 13456, :server => :server_local, :log => "/tmp/drbqs_execute_log"
+      def default(val = {})
+        val.delete_if { |key, v| !v }
+        @__default__.merge!(val)
+        if @__default__[:server]
+          @__default__[:server] = @__default__[:server].intern
+        end
+        if @__default__[:port]
+          @__default__[:port] = @__default__[:port].to_i
+        end
+      end
+
+      def default_clear(key)
+        @__default__.delete(key)
+      end
+
+      # We can set messages of usage and path of definition file of server
+      # to output help of server on showing help.
+      # * Example of usage
+      # usage(:message => 'Calculate some value', :server => 'server.rb')
+      def usage(opts = {})
+        @__usage__ = opts
+      end
+
+      def __load__(path)
+        instance_eval(File.read(path), path)
+      end
+    end
+  end
+end

data/lib/drbqs/{utility → execute}/server_define.rb

@@ -55,8 +55,8 @@ HELP
       @default_server_opts = nil
     end
 
-    def create_server(options)
-      server = DRbQS::Server.new(@default_server_opts.merge(options))
+    def create_server(options, klass = DRbQS::Server)
+      server = klass.new(@default_server_opts.merge(options))
       @server_create.call(server, @argv, @opts)
       server.set_signal_trap
       server
@@ -72,17 +72,10 @@ HELP
       server.wait
     end
 
-    def test_server(options, type, arg = [], test_opts = {})
-      server = create_server(options)
-      case type
-      when :task
-        puts "*** Test of Task Generators ***"
-        server.test_task_generator(:limit => arg[0] ? arg[0].to_i : nil, :progress => true)
-      when :exec
-        server.test_exec(:limit => arg[0] ? arg[0].to_i : nil, :profile => test_opts[:profile])
-      else
-        puts "*** Not be yet implemented ***"
-      end
+    def create_test_server(options)
+      require 'drbqs/server/test/server'
+      options[:finish_exit] = true
+      create_server(options, DRbQS::Test::Server)
     end
   end
 
@@ -90,7 +83,7 @@ HELP
 
   class << self
     [:define_server, :option_parser, :parse_option, :option_help_message, :clear_definition,
-     :start_server, :test_server].each do |m|
+     :start_server, :create_test_server].each do |m|
       define_method(m, &@@server_def.method(m))
     end
   end

data/lib/drbqs/manage/execute_node.rb

@@ -3,7 +3,7 @@ module DRbQS
   class ExecuteNode
     attr_reader :pid
 
-    def initialize(uri, log_prefix, log_level)
+    def initialize(uri, log_prefix, log_level, node_opts = {})
       @uri = uri
       @log_level = log_level
       if log_prefix
@@ -14,6 +14,7 @@ module DRbQS
         @fname = nil
       end
       @pid = []
+      @node_opts = node_opts
     end
 
     def get_log_file
@@ -26,7 +27,8 @@ module DRbQS
 
     def create_process
       @pid << fork do
-        node = DRbQS::Node.new(@uri, :log_level => @log_level, :log_file => get_log_file)
+        opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
+        node = DRbQS::Node.new(@uri, opts)
         node.connect
         node.calculate
       end

data/lib/drbqs/manage/manage.rb

@@ -1,13 +1,18 @@
+require 'forwardable'
 require 'drbqs/manage/ssh_execute'
 require 'drbqs/manage/send_signal'
 
 module DRbQS
   class Manage
+    extend Forwardable
+
     class NotSetURI < StandardError
     end
+    class NoServerRespond < StandardError
+    end
 
-    WAIT_SERVER_TIME = 0.3
-    WAIT_MAX_NUMBER = 200
+    WAIT_SERVER_TIME = 0.2
+    WAIT_MAX_NUMBER = 150
 
     # +opts+ has keys :home and :uri.
     def initialize(opts = {})
@@ -51,16 +56,9 @@ module DRbQS
       config.save_sample
     end
 
-    def send_exit_signal
-      signal_sender.send_exit_signal
-    end
-
-    def send_node_exit_after_task(node_id)
-      signal_sender.send_node_exit_after_task(node_id)
-    end
-
-    def get_status
-      signal_sender.get_status
+    [:get_status, :get_history, :send_exit_signal, :send_node_exit_after_task,
+     :send_node_wake, :send_node_sleep, :send_data].each do |method_name|
+      def_delegator :signal_sender, method_name, method_name
     end
 
     def server_respond?
@@ -74,16 +72,25 @@ module DRbQS
       end
     end
 
-    def wait_server_process(pid)
+    # If the server responds, this method returns true.
+    # If the server process does not exist, this method return nil.
+    # If the server process exists and there is no response,
+    # this method raises error.
+    def wait_server_process(pid = nil)
       i = 0
       begin
         sleep(WAIT_SERVER_TIME)
-        unless DRbQS::Misc.process_running_normally?(pid)
-          return nil
+        if pid
+          unless DRbQS::Misc.process_running_normally?(pid)
+            return nil
+          end
+        elsif server_data = config.list.server.get(@opts[:uri])
+          pid = server_data[:pid]
         end
         i += 1
         if i > WAIT_MAX_NUMBER
-          raise "Can not wait server process."
+          raise DRbQS::Manage::NoServerRespond,
+          "We are waiting for #{WAIT_SERVER_TIME * WAIT_MAX_NUMBER} seconds, but the server of #{@opts[:uri]} does not respond."
         end
       end while !server_respond?
       true

data/lib/drbqs/manage/send_signal.rb

@@ -24,13 +24,23 @@ module DRbQS
         send_signal_to_server(:exit_after_task, node_id)
       end
 
-      def get_status
-        send_signal_to_server(:request_status, sender_id)
+      def send_node_wake(node_id)
+        send_signal_to_server(:wake_node, node_id)
+      end
+
+      def send_node_sleep(node_id)
+        send_signal_to_server(:sleep_node, node_id)
+      end
+
+      def send_data(data)
+        send_signal_to_server(:new_data, data)
+      end
+
+      def wait_response(message_cond)
         i = 0
         loop do
           begin
-            mes = @message.take([:status, String], 0)
-            return mes[1]
+            return @message.take(message_cond, 0)
           rescue Rinda::RequestExpiredError
             i += 1
             if i > MAX_WAIT_TIME
@@ -40,6 +50,23 @@ module DRbQS
           end
         end
       end
+      private :wait_response
+
+      def get_status
+        send_signal_to_server(:request_status, sender_id)
+        if mes = wait_response([:status, String])
+          return mes[1]
+        end
+        nil
+      end
+
+      def get_history
+        send_signal_to_server(:request_history, sender_id)
+        if mes = wait_response([:history, String])
+          return mes[1]
+        end
+        nil
+      end
     end
   end
 end

data/lib/drbqs/manage/ssh_execute.rb

@@ -6,17 +6,61 @@ module DRbQS
       def initialize(dest, opts = {})
         @ssh_host = DRbQS::Config.new.ssh_host
         path, options = @ssh_host.get_options(dest)
-        # $stdout.puts "Use configuration: #{path}" if path
-        dest = options[:dest] || dest
-        @ssh = DRbQS::Manage::SSHShell.new(dest, options.merge(opts))
+        dest = options.delete(:dest) || dest
+        @ssh_shell = DRbQS::Manage::SSHShell.new(dest, options.merge(opts))
       end
 
-      def execute(command)
-        @ssh.start(command)
+      def command(command)
+        @ssh_shell.execute_all(command)
       end
 
       def get_environment
-        @ssh.get_environment
+        @ssh_shell.get_environment
+      end
+
+      # Keys are :nice and :daemon.
+      def add_command_options(cmd, daemon, nice)
+        if nice
+          s = "nice "
+          s << "-n #{nice.to_s} " if Integer === nice
+        else
+          s = ''
+        end
+        s << cmd << " --daemon " << daemon
+      end
+      private :add_command_options
+
+      def create_new_directory(sh, dir)
+        process, result = sh.exec("filename-create new -a always -D self -t time #{dir}", :check => true)
+        result.strip!
+      end
+      private :create_new_directory
+
+      # Add options --daemon and --log-file.
+      def server(cmd_options, opts = {})
+        ret = nil
+        @ssh_shell.start do |sh|
+          dir = create_new_directory(sh, opts[:daemon] || 'drbqs_server_log')
+          cmd = add_command_options('drbqs-server', File.join(dir, 'daemon_server.log'), opts[:nice])
+          cmd << " --log-file " << File.join(dir, 'server.log') << ' '
+          cmd << cmd_options.join(' ')
+          process, result = sh.exec(cmd)
+          ret = (process.exit_status == 0)
+        end
+        ret
+      end
+
+      # Add options --daemon and --log-prefix.
+      def node(cmd_options, opts = {})
+        ret = nil
+        @ssh_shell.start do |sh|
+          dir = create_new_directory(sh, opts[:daemon] || 'drbqs_node_log')
+          cmd = add_command_options('drbqs-node', File.join(dir, 'daemon_node.log'), opts[:nice])
+          cmd << ' --log-prefix ' << File.join(dir, 'node') << ' ' << cmd_options.join(' ')
+          process, result = sh.exec(cmd)
+          ret = (process.exit_status == 0)
+        end
+        ret
       end
     end
   end