drbqs 0.0.14 → 0.0.15

data/lib/drbqs/server/node_list.rb

@@ -9,6 +9,7 @@ module DRbQS
         @id = 0
         @list = {}
         @check = []
+        @prepare_to_exit = []
         @history = DRbQS::Server::History.new
       end
 
@@ -27,14 +28,15 @@ module DRbQS
         @check = @list.keys
       end
 
-      def delete(id)
+      def delete(id, history_state)
         @list.delete(id)
-        @history.set(id, :disconnect)
+        @prepare_to_exit.delete(id)
+        @history.set(id, history_state)
       end
 
       def delete_not_alive
         @check.each do |id|
-          delete(id)
+          delete(id, :disconnect)
         end
         deleted = @check
         @check = []
@@ -52,6 +54,17 @@ module DRbQS
       def exist?(id)
         @list.find { |a| a[0] == id }
       end
+
+      def prepare_to_exit?(node_id)
+        @prepare_to_exit.include?(node_id)
+      end
+
+      def add_to_preparation_to_exit(node_id)
+        unless prepare_to_exit?(node_id)
+          @history.set(node_id, :set_exitting)
+          @prepare_to_exit << node_id
+        end
+      end
     end
 
   end

data/lib/drbqs/server/prof.rb

@@ -0,0 +1,48 @@
+require 'ruby-prof'
+
+module DRbQS
+  class Prof
+    PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
+
+    # :flat
+    # :graph
+    # :graphhtml
+    # :calltree
+    def initialize(printer_type, output)
+      @printer_type = printer_type
+      unless PRINTER_TYPE.include?(@printer_type)
+        raise "Invalid printer type: #{@printer_type.inspect}"
+      end
+      @output = output
+    end
+
+    def get_printer(result)
+      case @printer_type
+      when :flat
+        RubyProf::FlatPrinter.new(result)
+      when :graph
+        RubyProf::GraphPrinter.new(result)
+      when :graphhtml
+        RubyProf::GraphHtmlPrinter.new(result)
+      when :calltree
+        RubyProf::CallTreePrinter.new(result)
+      end
+    end
+    private :get_printer
+
+    def start
+      RubyProf.start
+    end
+
+    def finish
+      printer = get_printer(RubyProf.stop)
+      if IO === @output
+        printer.print(@output)
+      else
+        Kernel.open(@output, 'w') do |f|
+          printer.print(f)
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/server/queue.rb

@@ -12,7 +12,7 @@ module DRbQS
         @task_id = 0
         @cache = {}
         @calculating = Hash.new { |hash, key| hash[key] = Array.new }
-        @history = DRbQS::Server::History.new
+        @history = DRbQS::Server::TaskHistory.new
         @logger = logger
       end
 
@@ -105,12 +105,20 @@ module DRbQS
         @calculating.inject(0) { |s, key_val| s + key_val[1].size }
       end
 
+      def stocked_task_number
+        @cache.size - calculating_task_number
+      end
+
+      def finished_task_number
+        @history.finished_task_number
+      end
+
       # If queue is empty, that is, there is no tasks to calculate next,
       # this method returns true. Otherwise, false.
       # Even if there are calculating tasks,
       # the method can return true.
       def empty?
-        @cache.size - calculating_task_number == 0
+        stocked_task_number == 0
       end
 
       # If there are no tasks in queue and calculating,
@@ -122,6 +130,16 @@ module DRbQS
       def all_logs
         @history.log_strings
       end
+
+      def calculating_nodes
+        nodes = []
+        @calculating.each do |node_id, tasks|
+          if tasks.size > 0
+            nodes << node_id
+          end
+        end
+        nodes.sort!
+      end
     end
 
   end

data/lib/drbqs/server/server.rb

@@ -32,6 +32,8 @@ module DRbQS
     #   Set the time interval of checking alive nodes.
     # :finish_exit
     #   Exit programs in finish_hook.
+    # :shutdown_unused_nodes
+    #   Shutdown unused nodes.
     # :signal_trap
     #   Set trapping signal.
     # :sftp_user
@@ -56,9 +58,10 @@ module DRbQS
       @queue= DRbQS::Server::Queue.new(@ts[:queue], @ts[:result], @logger)
       @check_alive = DRbQS::Server::CheckAlive.new(opts[:check_alive])
       @task_generator = []
-      hook_init(opts[:finish_exit])
+      hook_init(opts[:finish_exit], opts[:shutdown_unused_nodes])
       set_signal_trap if opts[:signal_trap]
       @finalization_task = nil
+      @data_storage = []
       @transfer_setting = DRbQS::Server::TransferSetting.new(opts[:sftp_host], opts[:sftp_user], opts[:file_directory])
       @config = DRbQS::Config.new
     end
@@ -79,9 +82,10 @@ module DRbQS
     end
     private :acl_init
 
-    def hook_init(finish_exit)
+    def hook_init(finish_exit, shutdown_nodes)
       @hook = DRbQS::Server::Hook.new
       @hook.set_finish_exit { self.exit } if finish_exit
+      @hook.set_shutdown_unused_nodes { shutdown_unused_nodes } if shutdown_nodes
     end
     private :hook_init
 
@@ -112,6 +116,8 @@ module DRbQS
       @task_generator << task_generator
     end
 
+    # If current task generator waits for finish of created tasks,
+    # this method returns true.
     def generator_waiting?
       @task_generator.size > 0 && @task_generator[0].waiting?
     end
@@ -134,6 +140,11 @@ module DRbQS
     end
     private :add_tasks_from_generator
 
+    def all_tasks_assigned?
+      @task_generator.empty? && @queue.empty?
+    end
+    private :all_tasks_assigned?
+
     def set_initialization_task(task)
       @message.set_initialization(task)
     end
@@ -143,9 +154,14 @@ module DRbQS
       @message.set_finalization(@finalization_task)
     end
 
-    # +key+ is :empty_queue or :finish_exit.
+    # +key+ is :empty_queue, :process_data, or :finish_exit.
     # &block takes self as an argument.
     def add_hook(key, name = nil, &block)
+      if key == :process_data
+        if @hook.number_of_hook(:process_data) != 0
+          raise "Hook :process_data has already set."
+        end
+      end
       @hook.add(key, name, &block)
     end
 
@@ -153,26 +169,67 @@ module DRbQS
       @hook.delete(key, name)
     end
 
+    def exec_empty_queue_hook
+      @hook.exec(:empty_queue, self) do |name|
+        if @queue.empty?
+          @logger.info("Execute empty queue hook: #{name}.")
+          true
+        else
+          false
+        end
+      end
+    end
+    private :exec_empty_queue_hook
+
     def exec_finish_hook
-      @logger.info("Execute finish hook.")
-      @hook.exec(:finish, self)
+      @hook.exec(:finish, self) do |name|
+        if !generator_waiting? && @queue.finished?
+          @logger.info("Execute finish hook: #{name}.")
+          true
+        else
+          false
+        end
+      end
     end
     private :exec_finish_hook
     
-    def exec_hook
-      if @queue.empty?
-        @logger.info("Execute empty queue hook.")
-        @hook.exec(:empty_queue, self)
+    def exec_task_assigned_hook
+      @hook.exec(:task_assigned, self) do |name|
+        if all_tasks_assigned?
+          @logger.info("Execute task assigned hook: #{name}.")
+          true
+        else
+          false
+        end
       end
+    end
+    private :exec_task_assigned_hook
+
+    def exec_process_data_hook
+      if @data_storage.size > 0
+        while data = @data_storage.shift
+          process_data(data)
+        end
+      end
+    end
+    private :exec_process_data_hook
+
+    def exec_hook
+      exec_process_data_hook
+      exec_empty_queue_hook
       if !generator_waiting? || @queue.finished?
         add_tasks_from_generator
       end
-      if !generator_waiting? && @queue.finished?
-        exec_finish_hook
-      end
+      exec_finish_hook
+      exec_task_assigned_hook
     end
     private :exec_hook
 
+    def shutdown_unused_nodes
+      @message.shutdown_unused_nodes(@queue.calculating_nodes)
+    end
+    private :shutdown_unused_nodes
+
     def exit
       if @finalization_task
         @message.send_finalization
@@ -203,19 +260,57 @@ module DRbQS
       end
     end
 
+    # Set *args to data storage, which must be string objects.
+    # The data is processed by hook of :process_data.
+    def set_data(*args)
+      args.each do |s|
+        if String === s
+          @data_storage << s
+        else
+          @logger.error("Invalid data type\n#{s.inspect}")
+        end
+      end
+    end
+
+    def process_data(data)
+      @hook.exec(:process_data, self, data)
+    rescue => err
+      @logger.error("Error in processing data.") do
+        "#{err.to_s} (#{err.class})\n#{err.backtrace.join("\n")}"
+      end
+    end
+    private :process_data
+
+    def send_status_for_request
+      data = {
+        :calculating_task_number => @queue.calculating_task_number,
+        :finished_task_number => @queue.finished_task_number,
+        :stocked_task_number => @queue.stocked_task_number,
+        :calculating_nodes => @queue.calculating,
+        :generator_number => @task_generator.size
+      }
+      @message.send_status(data)
+    end
+    private :send_status_for_request
+
     def check_message
       while mes_arg = @message.get_message
         mes, arg = mes_arg
         case mes
+        when :new_data
+          set_data(arg)
         when :exit_server
           self.exit
         when :request_status
-          @message.send_status(@queue.calculating)
+          send_status_for_request
+        when :request_history
+          @message.send_history(@queue.all_logs)
         when :exit_after_task
-          node_id = arg
-          if @message.node_exist?(node_id)
-            @message.send_exit_after_task(node_id)
-          end
+          @message.send_exit_after_task(arg)
+        when :wake_node
+          @message.send_wake(arg)
+        when :sleep_node
+          @message.send_sleep(arg)
         when :node_error
           @queue.get_accept_signal
           @queue.requeue_for_deleted_node_id([arg])
@@ -242,58 +337,5 @@ module DRbQS
         end
       end
     end
-
-    def start_profile
-      require 'ruby-prof'
-      RubyProf.start
-    end
-    private :start_profile
-
-    def finish_profile
-      result = RubyProf.stop
-      printer = RubyProf::FlatPrinter.new(result)
-      # printer = RubyProf::GraphPrinter.new(result)
-      # printer = RubyProf::CallTreePrinter.new(result)
-      printer.print(STDOUT)
-    end
-    private :finish_profile
-
-    def test_exec(opts = {})
-      first_task_generator_init
-      dummy_node = DRbQS::Node.new(nil, :log_file => $stdout, :log_level => opts[:log_level])
-      dummy_task_client = DRbQS::Node::TaskClient.new(nil, @ts[:queue], nil)
-      if @ts[:transfer]
-        dummy_node.instance_variable_set(:@transfer, DRbQS::TransferClient::Local.new(@ts[:transfer].directory))
-      end
-      num = 0
-      start_profile if opts[:profile]
-      loop do
-        exec_hook
-        if ary = dummy_task_client.get_task
-          task_id, marshal_obj, method_sym, args = ary
-          result = dummy_node.instance_eval { execute_task(marshal_obj, method_sym, args) }
-          @queue.exec_task_hook(self, task_id, result)
-        end
-        num += 1
-        if opts[:limit] && num >= opts[:limit]
-          break
-        end
-      end
-      finish_profile if opts[:profile]
-      if @finalization_task
-        args = @finalization_task.drb_args(nil)[1..-1]
-        dummy_node.instance_eval { execute_task(*args) }
-      end
-      exec_finish_hook
-    end
-
-    def test_task_generator(opts = {})
-      @task_generator.each_with_index do |t, i|
-        puts "Test task generator [#{i}]"
-        t.init
-        set_num, task_num = t.debug_all_tasks(opts)
-        puts "Create: task sets #{set_num}, all tasks #{task_num}"
-      end
-    end
   end
 end

data/lib/drbqs/server/server_hook.rb

@@ -6,7 +6,9 @@ module DRbQS
         @argument_number = {}
         @finish_exit = nil
         set_argument_number(:empty_queue, 1)
+        set_argument_number(:process_data, 2)
         set_argument_number(:finish, 1)
+        set_argument_number(:task_assigned, 1)
       end
 
       def set_argument_number(key, num)
@@ -41,10 +43,20 @@ module DRbQS
         end
       end
 
-      def specific_proc(key)
+      def specific_proc(key, &cond)
         case key
         when :finish
-          @finish_exit.call if @finish_exit
+          if @finish_exit
+            if !cond || cond.call('special:finish_exit')
+              @finish_exit.call
+            end
+          end
+        when :task_assigned
+          if @shutdown_unused_nodes
+            if !cond || cond.call('special:task_assigned')
+              @shutdown_unused_nodes.call
+            end
+          end
         end
       end
       private :specific_proc
@@ -53,19 +65,27 @@ module DRbQS
         @hook[key].map { |a| a[0] }
       end
 
-      def exec(key, *args)
+      def exec(key, *args, &cond)
         @hook[key].each do |ary|
-          ary[1].call(*args)
+          if !cond || cond.call(ary[0])
+            ary[1].call(*args)
+          else
+            return nil
+          end
         end
-        specific_proc(key)
+        specific_proc(key, &cond)
       end
 
       def set_finish_exit(&block)
         @finish_exit = block
       end
 
+      def set_shutdown_unused_nodes(&block)
+        @shutdown_unused_nodes = block
+      end
+
       def number_of_hook(key)
-        @hook[key].size
+        @hook.has_key?(key) ? @hook[key].size : 0
       end
     end
   end

data/lib/drbqs/server/test/node.rb

@@ -0,0 +1,34 @@
+require 'drbqs/node/node'
+
+module DRbQS
+  module Test
+    class Node < DRbQS::Node
+      def initialize(log_level, transfer, queue)
+        super(nil, :log_file => $stdout, :log_level => log_level)
+        @transfer = transfer
+        @task_client = DRbQS::Node::TaskClient.new(nil, queue, nil)
+      end
+
+      def server_on_same_host?
+        true
+      end
+
+      def calc
+        if ary = @task_client.get_task
+          task_id, marshal_obj, method_sym, args = ary
+          result = execute_task(marshal_obj, method_sym, args)
+          return [task_id, result]
+        end
+        nil
+      end
+
+      def finalize(finalization_task)
+        if finalization_task
+          args = finalization_task.drb_args(nil)[1..-1]
+          execute_task(*args)
+        end
+        clear_node_files
+      end
+    end
+  end
+end

data/lib/drbqs/server/test/server.rb

@@ -0,0 +1,74 @@
+module DRbQS
+  module Test
+    class Server < DRbQS::Server
+      PROF_FILE = 'drbqs_prof.txt'
+
+      def exit
+        throw(:exit_loop)
+      end
+
+      def loop_for_test(limit, profile, printer, &block)
+        result = { :start => Time.now }
+        num = 0
+        if profile
+          require 'drbqs/server/prof'
+          result[:profile] = FileName.create(PROF_FILE, :position => :middle)
+          prof = DRbQS::Prof.new(printer || :flat, result[:profile])
+          prof.start
+        end
+        begin
+          catch(:exit_loop) do
+            loop do
+              yield
+              if limit
+                num += 1
+                if num >= limit
+                  exec_finish_hook
+                  break
+                end
+              end
+            end
+          end
+        rescue Exception => err
+          $stdout.puts "*** Error occurs in calculation roop ***"
+          b = err.backtrace
+          $stdout.puts "#{b[0]}: #{err.to_s} (#{err.class})"
+          $stdout.puts b[1..-1].join("\n") if b.size > 1
+        end
+        if profile
+          prof.finish
+        end
+        result[:end] = Time.now
+        result
+      end
+      private :loop_for_test
+
+      def test_exec(opts = {})
+        require 'drbqs/server/test/node'
+        first_task_generator_init
+        set_file_transfer(nil)
+        test_node = DRbQS::Test::Node.new(@logger.level, @ts[:transfer], @ts[:queue])
+        n = 0
+        data = loop_for_test(opts[:limit], opts[:profile], opts[:printer]) do
+          exec_hook
+          if ary = test_node.calc
+            @queue.exec_task_hook(self, *ary)
+            n += 1
+          end
+        end
+        test_node.finalize(@finalization_task)
+        data[:task] = n
+        data
+      end
+
+      def test_task_generator(opts = {})
+        @task_generator.each_with_index do |t, i|
+          puts "Test task generator [#{i}]"
+          t.init
+          set_num, task_num = t.debug_all_tasks(opts)
+          puts "Create: task sets #{set_num}, all tasks #{task_num}"
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/setting/base.rb

@@ -0,0 +1,120 @@
+require 'forwardable'
+
+module DRbQS
+  class Setting
+    class InvalidLogLevel < StandardError
+    end
+
+    LOG_LEVEL_DEFAULT = Logger::ERROR
+
+    # A base class having options of commands.
+    # We must define a method 'exec' this method in a child class.
+    class Base
+      extend Forwardable
+
+      attr_reader :source
+
+      # The keys of options are :all_keys_defined, :log_level, and :daemon
+      def initialize(opts = {}, &block)
+        @source = DRbQS::Setting::Source.new(opts[:all_keys_defined])
+        @source.register_key(:debug, :bool => true)
+        if opts[:log_level]
+          @source.register_key(:log_level, :check => 1, :default => [Logger::ERROR])
+        end
+        if opts[:daemon]
+          @__daemon__ = nil
+          @source.register_key(:daemon, :check => 1)
+        end
+        @source.instance_eval(&block) if block_given?
+        @options = {}
+      end
+
+      def_delegator :@source, :set, :set
+      def_delegator :@source, :get, :get
+      def_delegator :@source, :clear, :clear
+      def_delegator :@source, :set?, :set?
+      def_delegator :@source, :get_first, :get_first
+      def_delegator :@source, :set_argument, :set_argument
+      def_delegator :@source, :get_argument, :get_argument
+      def_delegator :@source, :command_line_argument, :command_line_argument
+      def_delegator :@source, :default, :default
+
+      def string_for_shell
+        command_line_argument(true).join(" ")
+      end
+
+      def value
+        @source.value
+      end
+
+      def preprocess!
+      end
+      private :preprocess!
+
+      # We execute DRbQS::Setting::Base#parse! before execute 'exec'.
+      def parse!
+        preprocess!
+        @source.check!
+        $DEBUG = true if get_first(:debug)
+        @__daemon__ = get_first(:daemon)
+        parse_log_level
+      end
+
+      def parse_log_level
+        if arg = get_first(:log_level)
+          case arg
+          when /^(fatal)|(error)|(warn)|(info)|(debug)|(unknown)$/i
+            n = eval("Logger::#{arg.upcase}")
+            unless 0 <= n && n <= 5
+              raise DRbQS::Setting::InvalidLogLevel, "error: Invalid log level '#{arg}'"
+            end
+          when /^[0-5]$/
+            n = arg.to_i
+          when 0..5
+            n = arg
+          else
+            raise DRbQS::Setting::InvalidLogLevel, "error: Invalid log level '#{arg}'"
+          end
+          @options[:log_level] = n
+        end
+      end
+      private :parse_log_level
+
+      def daemon_start(output, &block)
+        Process.daemon(true)
+        begin
+          $stdout = Kernel.open(output, 'w')
+          $stderr = $stdout
+          begin
+            yield
+          rescue SystemExit
+            return true
+          end
+        rescue Exception => err
+          output_error(err)
+        ensure
+          $stdout.close
+        end
+      end
+      private :daemon_start
+
+      def exec_as_daemon(&block)
+        if @__daemon__
+          @__daemon__ = FileName.create(@__daemon__, :position => :middle, :type => :time, :directory => :parent)
+          daemon_start(@__daemon__) do
+            @__daemon__ = nil
+            if block_given?
+              yield
+            else
+              exec($stdout)
+            end
+          end
+          true
+        else
+          nil
+        end
+      end
+      private :exec_as_daemon
+    end
+  end
+end