drbqs 0.0.15 → 0.0.16

data/lib/drbqs/execute/server_define.rb

@@ -1,85 +1,96 @@
 module DRbQS
-
-  class ServerDefinition
-    HELP_MESSAGE =<<HELP
+  class Execution
+    class ServerDefinition
+      HELP_MESSAGE =<<HELP
 * Server specific options
   These options are separated by '--' from command options.
 
 HELP
 
-    def initialize
-      clear_definition
-    end
+      def initialize
+        clear_definition
+      end
 
-    def define_server(default_opts = {}, &block)
-      @default_server_opts = default_opts
-      if @server_create
-        raise ArgumentError, "The server has already defined."
+      def define_server(default_opts = {}, &block)
+        @default_server_opts = default_opts
+        if @server_create
+          raise ArgumentError, "The server has already defined."
+        end
+        @server_create = block
       end
-      @server_create = block
-    end
 
-    def option_parser(&block)
-      if @option_parse
-        raise ArgumentError, "The options parser has already defined."
+      def option_parser(mes = nil, &block)
+        if @option_parse
+          raise ArgumentError, "The options parser has already defined."
+        end
+        @option_message = mes
+        @option_parse = block
       end
-      @option_parse = block
-    end
 
-    def parse_option(opt_argv)
-      if @option_parse
-        OptionParser.new(HELP_MESSAGE) do |opt|
-          @option_parse.call(opt, @opts)
-          opt.parse!(opt_argv)
+      def create_parser(&block)
+        mes = HELP_MESSAGE
+        mes += @option_message.gsub(/(^\n+)|(\n+$)/, '') + "\n\n" if @option_message
+        OptionParser.new(mes) do |opt|
+          yield(opt)
         end
       end
-      @argv = opt_argv  
-    end
+      private :create_parser
 
-    def option_help_message
-      if @option_parse
-        OptionParser.new(HELP_MESSAGE) do |opt|
-          @option_parse.call(opt, @opts)
-          return opt.to_s
+      def parse_option(opt_argv)
+        if @option_parse
+          create_parser do |opt|
+            @option_parse.call(opt, @opts)
+            opt.parse!(opt_argv)
+          end
         end
-      else
-        nil
+        @argv = opt_argv  
       end
-    end
 
-    def clear_definition
-      @server_create = nil
-      @option_parse = nil
-      @opts = {}
-      @argv = nil
-      @default_server_opts = nil
-    end
+      def option_help_message
+        if @option_parse
+          create_parser do |opt|
+            @option_parse.call(opt, @opts)
+            return opt.to_s
+          end
+        else
+          nil
+        end
+      end
 
-    def create_server(options, klass = DRbQS::Server)
-      server = klass.new(@default_server_opts.merge(options))
-      @server_create.call(server, @argv, @opts)
-      server.set_signal_trap
-      server
-    end
-    private :create_server
+      def clear_definition
+        @server_create = nil
+        @option_parse = nil
+        @opts = {}
+        @argv = nil
+        @default_server_opts = nil
+      end
 
-    def start_server(options)
-      unless @server_create
-        raise "Can not get server definition."
+      def create_server(options, klass = DRbQS::Server)
+        server = klass.new(@default_server_opts.merge(options))
+        @server_create.call(server, @argv, @opts)
+        server.set_signal_trap
+        server
       end
-      server = create_server(options)
-      server.start
-      server.wait
-    end
+      private :create_server
 
-    def create_test_server(options)
-      require 'drbqs/server/test/server'
-      options[:finish_exit] = true
-      create_server(options, DRbQS::Test::Server)
+      def start_server(options)
+        unless @server_create
+          raise "Can not get server definition."
+        end
+        server = create_server(options)
+        server.start
+        server.wait
+      end
+
+      def create_test_server(options)
+        require 'drbqs/server/test/server'
+        options.delete(:not_exit)
+        create_server(options, DRbQS::Test::Server)
+      end
     end
   end
 
-  @@server_def = DRbQS::ServerDefinition.new
+  @@server_def = DRbQS::Execution::ServerDefinition.new
 
   class << self
     [:define_server, :option_parser, :parse_option, :option_help_message, :clear_definition,

data/lib/drbqs/ext/task.rb

@@ -0,0 +1,2 @@
+require 'drbqs/task/task'
+require 'drbqs/ext/task/command_task.rb'

data/lib/drbqs/ext/task/command_task.rb

@@ -0,0 +1,43 @@
+module DRbQS
+  # Class to define tasks such that we execute a command.
+  class CommandTask < DRbQS::Task
+
+    # Execute a command and transfer files if needed.
+    class CommandExecute
+
+      # :transfer    String or Array
+      # :compress    true or false
+      def initialize(cmd, opts = {})
+        @cmd = cmd
+        unless (Array === @cmd || String === @cmd)
+          raise ArgumentError, "Invalid command: #{@cmd.inspect}"
+        end
+        @transfer = opts[:transfer]
+        @compress = opts[:compress]
+      end
+
+      def exec
+        case @cmd
+        when Array
+          @cmd.each { |c| system(c) }
+        when String
+          system(@cmd)
+        end
+        exit_status = $?.exitstatus
+        if @transfer
+          if @transfer.respond_to?(:each)
+            @transfer.each { |path| DRbQS::Transfer.enqueue(path, :compress => @compress) }
+          else
+            DRbQS::Transfer.enqueue(@transfer, :compress => @compress)
+          end
+        end
+        exit_status
+      end
+    end
+
+    # &hook takes a server instance and exit number of command.
+    def initialize(cmd, opts = {}, &hook)
+      super(DRbQS::CommandTask::CommandExecute.new(cmd, opts), :exec, &hook)
+    end
+  end
+end

data/lib/drbqs/manage/manage.rb

@@ -4,12 +4,13 @@ require 'drbqs/manage/send_signal'
 
 module DRbQS
   class Manage
-    extend Forwardable
+    class NoServerRespond < StandardError
+    end
 
     class NotSetURI < StandardError
     end
-    class NoServerRespond < StandardError
-    end
+
+    extend Forwardable
 
     WAIT_SERVER_TIME = 0.2
     WAIT_MAX_NUMBER = 150
@@ -43,7 +44,7 @@ module DRbQS
     def signal_sender
       unless @signal_sender
         unless @opts[:uri]
-          raise DRbQS::Manage::NotSetURI, "The uri has not set yet."
+          raise DRbQS::Manage::NotSetURI, "The uri of server to connect has not set."
         end
         obj = DRbObject.new_with_uri(@opts[:uri])
         @signal_sender = DRbQS::Manage::SendSignal.new(obj[:message])

data/lib/drbqs/manage/ssh_shell.rb

@@ -2,7 +2,6 @@ require 'net/ssh'
 require 'net/ssh/shell'
 
 module DRbQS
-
   class Manage
     # Requirements:
     #   bash
@@ -53,11 +52,6 @@ module DRbQS
         end
       end
 
-      class InvalidDestination < StandardError
-      end
-      class GetInvalidExitStatus < StandardError
-      end
-
       attr_reader :user, :host, :port, :keys
 
       # :shell     shell to use
@@ -70,7 +64,7 @@ module DRbQS
       def initialize(dest, opts = {})
         @user, @host, @port = split_destination(dest)
         if !(@host && @user)
-          raise InvalidDestination, "Invalid destination of ssh server."
+          raise ArgumentError, "Invalid ssh server: host=#{@host.inspect}, user=#{@user.inspect}."
         end
         @keys = opts.delete(:keys)
         @shell = opts[:shell] || 'bash'
@@ -137,7 +131,7 @@ module DRbQS
         ary = shell_exec(sh, cmd)
         n = ary[0].exit_status
         if n != 0
-          raise GetInvalidExitStatus, "Can not execute properly on #{@host}.\nExit status: #{n}\ncommand: #{cmd}"
+          raise RuntimeError, "Can not execute properly on #{@host}.\nExit status: #{n}\ncommand: #{cmd}"
         end
         ary
       end

data/lib/drbqs/node/connection.rb

@@ -4,7 +4,7 @@ module DRbQS
     class Connection
       attr_reader :id, :node_number
 
-      def initialize(message, logger = DRbQS::LoggerDummy.new)
+      def initialize(message, logger = DRbQS::Misc::LoggerDummy.new)
         @message = message
         @logger = logger
         @node_number = nil

data/lib/drbqs/node/node.rb

@@ -1,6 +1,6 @@
 require 'drbqs/task/task'
-require 'drbqs/utility/transfer/transfer_client'
 require 'drbqs/utility/temporary'
+require 'drbqs/utility/transfer/transfer_client'
 require 'drbqs/node/connection'
 require 'drbqs/node/task_client'
 require 'drbqs/node/state'
@@ -30,19 +30,13 @@ module DRbQS
     end
 
     def transfer_file
-      if files = DRbQS::FileTransfer.dequeue_all
-        if @transfer
-          begin
-            @transfer.transfer(files, server_on_same_host?)
-          rescue => err
-            @logger.error("Fail to transfer files.") do
-              "Can not send file: #{files.join(", ")}\n#{err.to_s}\n#{err.backtrace.join("\n")}"
-            end
-            raise
-          end
-        else
-          raise "Server does not set transfer settings. Can not send file: #{files.join(", ")}"
+      begin
+        DRbQS::Transfer::Client.transfer_to_server
+      rescue Exception => err
+        @logger.error("Fail to transfer files.") do
+          "#{err.to_s} (#{err.class})\n#{err.backtrace.join("\n")}"
         end
+        raise
       end
     end
     private :transfer_file
@@ -65,7 +59,7 @@ module DRbQS
       @server_key = obj[:key]
       @connection = Node::Connection.new(obj[:message], @logger)
       @task_client = Node::TaskClient.new(@connection.node_number, obj[:queue], obj[:result], @logger)
-      @transfer = obj[:transfer]
+      DRbQS::Transfer::Client.set(obj[:transfer].get_client(server_on_same_host?)) if obj[:transfer]
       if ary = @connection.get_initialization
         execute_task(*ary)
       end

data/lib/drbqs/node/task_client.rb

@@ -3,7 +3,7 @@ module DRbQS
     class TaskClient
       attr_reader :node_number, :calculating_task
 
-      def initialize(node_number, queue, result, logger = DRbQS::LoggerDummy.new)
+      def initialize(node_number, queue, result, logger = DRbQS::Misc::LoggerDummy.new)
         @node_number = node_number
         @queue = queue
         @result = result

data/lib/drbqs/server/history.rb

@@ -51,7 +51,11 @@ module DRbQS
           s << "Task #{task_id}\n"
           events.each do |ev|
             case ev[1]
-            when :add, :requeue, :hook
+            when :add
+              s << "  #{time_to_history_string(ev[0])}\t#{ev[1]}"
+              s << "\t" << ev[2].to_s if ev[2]
+              s << "\n"
+            when :requeue, :hook
               s << "  #{time_to_history_string(ev[0])}\t#{ev[1]}\n"
             when :calculate, :result
               s << "  #{time_to_history_string(ev[0])}\t#{ev[1]} (node #{ev[2]})\n"

data/lib/drbqs/server/message.rb

@@ -5,7 +5,7 @@ module DRbQS
     class Message
       include DRbQS::Misc
 
-      def initialize(message, logger = DRbQS::LoggerDummy.new)
+      def initialize(message, logger = DRbQS::Misc::LoggerDummy.new)
         @message = message
         @node_list = DRbQS::Server::NodeList.new
         @logger = logger
@@ -113,43 +113,16 @@ module DRbQS
         send_signal(node_id, :exit_after_task)
       end
 
-      # +data+ is a hash including server information.
-      # The keys are :calculating_task_number, :finished_task_number, :stocked_task_number,
-      # :calculating_nodes, and :generator_number.
-      def send_status(data)
-        s = "Nodes:\n"
-        if @node_list.history.size == 0
-          s << "  none\n"
-        else
-          @node_list.history.each do |node_id, events|
-            if events.size == 0
-              s << "Empty history of node #{node_id}\n"
-            else
-              connect = events[0]
-              s << sprintf("%4d %s\t", node_id, connect[2])
-              if events.size > 1
-                s << "start:#{time_to_history_string(connect[0])}"
-                events[1..-1].each do |t, key|
-                  s << ", #{key}: #{time_to_history_string(t)}"
-                end
-                s << "\n"
-              elsif data[:calculating_nodes]
-                task_ids = data[:calculating_nodes][node_id].to_a
-                s << "task: #{task_ids.map { |num| num.to_s }.join(', ')} (#{time_to_history_string(connect[0])})\n"
-              end
-            end
-          end
-        end
-        s << "Server:\n"
-        s << "  calculating tasks: #{data[:calculating_task_number]}\n"
-        s << "  finished tasks   : #{data[:finished_task_number]}\n"
-        s << "  stocked tasks    : #{data[:stocked_task_number]}\n"
-        s << "  task generator   : #{data[:generator_number]}"
+      def each_node_history(&block)
+        @node_list.history.each(&block)
+      end
+
+      def send_status(server_status_string)
         begin
           @message.take([:status, nil], 0)
         rescue Rinda::RequestExpiredError
         end
-        @message.write([:status, s])
+        @message.write([:status, server_status_string])
       end
 
       def send_history(history_string)

data/lib/drbqs/server/queue.rb

@@ -6,7 +6,7 @@ module DRbQS
     class Queue
       attr_reader :calculating, :history
 
-      def initialize(queue, result, logger = DRbQS::LoggerDummy.new)
+      def initialize(queue, result, logger = DRbQS::Misc::LoggerDummy.new)
         @queue = queue
         @result = result
         @task_id = 0
@@ -28,7 +28,7 @@ module DRbQS
         @logger.info("New task: #{@task_id}")
         @cache[@task_id] = task
         queue_task(@task_id)
-        @history.set(@task_id, :add)
+        @history.set(@task_id, :add, task.note)
         @task_id
       end
 
@@ -101,6 +101,18 @@ module DRbQS
         count
       end
 
+      # Return a hash of which keys are node ID number and
+      # values are an array of pairs of task ID number and its message.
+      def calculating_task_message
+        mes = {}
+        @calculating.each do |node_id, task_id_ary|
+          mes[node_id] = task_id_ary.map do |n|
+            [n, @history.events(n)[0][2]]
+          end
+        end
+        mes
+      end
+
       def calculating_task_number
         @calculating.inject(0) { |s, key_val| s + key_val[1].size }
       end

data/lib/drbqs/server/server.rb

@@ -8,40 +8,28 @@ require 'drbqs/server/server_hook'
 
 module DRbQS
 
-  # When we set both empty_queue_hook and task_generator,
-  # empty_queue_hook is prior to task_generator.
   class Server
+    include DRbQS::Misc
+
     WAIT_TIME_NODE_EXIT = 3
     WAIT_TIME_NODE_FINALIZE = 10
     WAIT_TIME_NEW_RESULT = 1
 
     attr_reader :queue, :uri
 
-    # :port
-    #   Set the port of server.
-    # :unix
-    #   Set the path of unix domain socket.
-    #   If :port is specified, :port is preceded.
-    # :acl
-    #   Set the ACL instance.
-    # :log_file
-    #   Set the path of log files.
-    # :log_level
-    #   Set the level of logging.
-    # :check_alive
-    #   Set the time interval of checking alive nodes.
-    # :finish_exit
-    #   Exit programs in finish_hook.
-    # :shutdown_unused_nodes
-    #   Shutdown unused nodes.
-    # :signal_trap
-    #   Set trapping signal.
-    # :sftp_user
-    #   Set user of sftp.
-    # :sftp_host
-    #   Set host of sftp.
-    # :file_directory
-    #   Set the setting of file directory.
+    # @param [Hash] opts The options of server
+    # @option opts [Hash] :port Set the port of server.
+    # @option opts [Hash] :unix Set the path of unix domain socket. If :port is specified then :port is preceded.
+    # @option opts [Hash] :acl Set the ACL instance.
+    # @option opts [Hash] :log_file Set the path of log files.
+    # @option opts [Hash] :log_level Set the level of logging.
+    # @option opts [Hash] :check_alive Set the time interval of checking alive nodes.
+    # @option opts [Hash] :not_exit Not exit programs when all tasks are finished.
+    # @option opts [Hash] :shutdown_unused_nodes Shutdown unused nodes.
+    # @option opts [Hash] :signal_trap Set trapping signal. Default is true.
+    # @option opts [Hash] :sftp_user Set user of sftp.
+    # @option opts [Hash] :sftp_host Set host of sftp.
+    # @option opts [Hash] :file_directory Set the directory for nodes to send files.
     def initialize(opts = {})
       @uri = DRbQS::Misc.create_uri(opts)
       @acl = acl_init(opts[:acl])
@@ -58,8 +46,8 @@ module DRbQS
       @queue= DRbQS::Server::Queue.new(@ts[:queue], @ts[:result], @logger)
       @check_alive = DRbQS::Server::CheckAlive.new(opts[:check_alive])
       @task_generator = []
-      hook_init(opts[:finish_exit], opts[:shutdown_unused_nodes])
-      set_signal_trap if opts[:signal_trap]
+      hook_init(!opts[:not_exit], opts[:shutdown_unused_nodes])
+      set_signal_trap if !opts.has_key?(:signal_trap) || opts[:signal_trap]
       @finalization_task = nil
       @data_storage = []
       @transfer_setting = DRbQS::Server::TransferSetting.new(opts[:sftp_host], opts[:sftp_user], opts[:file_directory])
@@ -67,7 +55,7 @@ module DRbQS
     end
 
     def transfer_directory
-      @ts[:transfer] && @ts[:transfer].directory
+      @transfer_setting.prepared_directory
     end
 
     def acl_init(acl_arg)
@@ -94,6 +82,7 @@ module DRbQS
     end
     private :server_data
 
+    # Initialize and start druby service.
     def start
       set_file_transfer(nil)
       DRb.install_acl(@acl) if @acl
@@ -112,10 +101,20 @@ module DRbQS
     end
     private :check_connection
 
+    # @param [DRbQS::Task::Generator] task_generator
     def add_task_generator(task_generator)
       @task_generator << task_generator
     end
 
+    # Create new task generator and add it.
+    # The arguments are same as {DRbQS::Task::Generator#set}.
+    def task_generator(opts = {}, &block)
+      gen = DRbQS::Task::Generator.new
+      gen.set(opts, &block)
+      add_task_generator(gen)
+      nil
+    end
+
     # If current task generator waits for finish of created tasks,
     # this method returns true.
     def generator_waiting?
@@ -145,17 +144,23 @@ module DRbQS
     end
     private :all_tasks_assigned?
 
+    # @param [DRbQS::task] task
     def set_initialization_task(task)
       @message.set_initialization(task)
     end
 
+    # @param [DRbQS::task] task
     def set_finalization_task(task)
       @finalization_task = task
       @message.set_finalization(@finalization_task)
     end
 
-    # +key+ is :empty_queue, :process_data, or :finish_exit.
-    # &block takes self as an argument.
+    # Set a hook of server.
+    # @note When we set both :empty_queue and task generators,
+    #  hook of :empty_queue is prior to task generators.
+    # @param [:empty_queue,:process_data,:finish] key Set the type of hook.
+    # @param [String] name Name of the hook. If the value is nil then the name is automatically created.
+    # @param [Proc] &block block is obligatory and takes server itself as an argument.
     def add_hook(key, name = nil, &block)
       if key == :process_data
         if @hook.number_of_hook(:process_data) != 0
@@ -165,6 +170,8 @@ module DRbQS
       @hook.add(key, name, &block)
     end
 
+    # @param [:empty_queue,:process_data,:finish] key Set the type of hook.
+    # @param [String] name Name of the hook. If the value is nil then all hooks of the key is deleted.
     def delete_hook(key, name = nil)
       @hook.delete(key, name)
     end
@@ -249,19 +256,25 @@ module DRbQS
 
     def set_signal_trap
       Signal.trap(:TERM) do
+        @logger.error("Get TERM signal.")
         self.exit
       end
     end
 
+    # @param [String] directory Set the directory to save files from nodes.
+    # @param [Hash] opts The options for SFTP.
+    # @option opts [Symbol] :host Hostname for SFTP.
+    # @option opts [Symbol] :user User name for SFTP.
     def set_file_transfer(directory, opts = {})
-      if transfer = @transfer_setting.create(directory, opts)
-        @ts[:transfer] = transfer
-        @logger.info("File transfer") { transfer.information }
+      if @transfer_setting.setup_server(directory, opts)
+        @ts[:transfer] = @transfer_setting
+        @logger.info("File transfer") { @transfer_setting.information }
       end
     end
 
     # Set *args to data storage, which must be string objects.
     # The data is processed by hook of :process_data.
+    # @param [Array] args An array of data strings.
     def set_data(*args)
       args.each do |s|
         if String === s
@@ -282,14 +295,44 @@ module DRbQS
     private :process_data
 
     def send_status_for_request
-      data = {
-        :calculating_task_number => @queue.calculating_task_number,
-        :finished_task_number => @queue.finished_task_number,
-        :stocked_task_number => @queue.stocked_task_number,
-        :calculating_nodes => @queue.calculating,
-        :generator_number => @task_generator.size
-      }
-      @message.send_status(data)
+      messages = @queue.calculating_task_message
+      s = ''
+      @message.each_node_history do|node_id, events|
+        if events.size == 0
+          s << "Empty history of node #{node_id}\n"
+        else
+          connect = events[0]
+          s << sprintf("%4d %s\t", node_id, connect[2])
+          if events.size > 1
+            s << "start:#{time_to_history_string2(connect[0])}"
+            events[1..-1].each do |t, key|
+              s << ", #{key}: #{time_to_history_string2(t)}"
+            end
+            s << "\n"
+          else
+            task_ids = @queue.calculating[node_id].to_a
+            s << "task: "
+            if messages[node_id]
+              s << messages[node_id].map do |task_id, mes|
+                calc_task = task_id.to_s
+                calc_task << ": " << mes.to_s if mes
+                calc_task
+              end.join(', ')
+            else
+              s << "none"
+            end
+            s << " (#{time_to_history_string2(connect[0])})\n"
+          end
+        end
+      end
+      s << "  none\n" if s.size == 0
+      s = "Nodes:\n" << s
+      s << "Server:\n"
+      s << "  calculating tasks: #{@queue.calculating_task_number}\n"
+      s << "  finished tasks   : #{@queue.finished_task_number}\n"
+      s << "  stocked tasks    : #{@queue.stocked_task_number}\n"
+      s << "  task generator   : #{@task_generator.size}"
+      @message.send_status(s)
     end
     private :send_status_for_request