drbqs 0.0.15 → 0.0.16

data/lib/drbqs/server/server_hook.rb

@@ -27,6 +27,9 @@ module DRbQS
       private :create_proc_name
 
       def add(key, name = nil, &block)
+        unless block_given?
+          raise ArgumentError, "The main part of hook must be specified as a block."
+        end
         if (n = @argument_number[key]) && (block.arity != n)
           raise ArgumentError, "Invalid argument number of hook of #{key.inspect}."
         end

data/lib/drbqs/server/test/node.rb

@@ -5,7 +5,7 @@ module DRbQS
     class Node < DRbQS::Node
       def initialize(log_level, transfer, queue)
         super(nil, :log_file => $stdout, :log_level => log_level)
-        @transfer = transfer
+        DRbQS::Transfer::Client.set(transfer.get_client(true)) if transfer
         @task_client = DRbQS::Node::TaskClient.new(nil, queue, nil)
       end
 

data/lib/drbqs/server/test/prof.rb

@@ -0,0 +1,50 @@
+require 'ruby-prof'
+
+module DRbQS
+  module Test
+    class Prof
+      PRINTER_TYPE = [:flat, :graph, :graphhtml, :calltree]
+
+      # :flat
+      # :graph
+      # :graphhtml
+      # :calltree
+      def initialize(printer_type, output)
+        @printer_type = printer_type
+        unless PRINTER_TYPE.include?(@printer_type)
+          raise "Invalid printer type: #{@printer_type.inspect}"
+        end
+        @output = output
+      end
+
+      def get_printer(result)
+        case @printer_type
+        when :flat
+          RubyProf::FlatPrinter.new(result)
+        when :graph
+          RubyProf::GraphPrinter.new(result)
+        when :graphhtml
+          RubyProf::GraphHtmlPrinter.new(result)
+        when :calltree
+          RubyProf::CallTreePrinter.new(result)
+        end
+      end
+      private :get_printer
+
+      def start
+        RubyProf.start
+      end
+
+      def finish
+        printer = get_printer(RubyProf.stop)
+        if IO === @output
+          printer.print(@output)
+        else
+          Kernel.open(@output, 'w') do |f|
+            printer.print(f)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/server/test/server.rb

@@ -11,9 +11,9 @@ module DRbQS
         result = { :start => Time.now }
         num = 0
         if profile
-          require 'drbqs/server/prof'
+          require 'drbqs/server/test/prof'
           result[:profile] = FileName.create(PROF_FILE, :position => :middle)
-          prof = DRbQS::Prof.new(printer || :flat, result[:profile])
+          prof = DRbQS::Test::Prof.new(printer || :flat, result[:profile])
           prof.start
         end
         begin

data/lib/drbqs/server/transfer_setting.rb

@@ -9,21 +9,33 @@ module DRbQS
         @host = host
         @user = user
         @directory = directory
-        @created = false
+        @setup_server = false
       end
 
-      def create(directory, opts = {})
-        return nil if @created
+      def prepared_directory
+        @setup_server && @directory
+      end
+
+      def information
+        info = "directory: #{@directory}"
+        info << ", sftp: #{@user}@#{@host}" if @host && @user
+        info
+      end
+
+      def setup_server(directory, opts = {})
+        return nil if @setup_server
         @directory = directory || @directory
         return nil if !@directory
-        @created = true
-        transfer_client = DRbQS::TransferClient.new(@directory)
-        transfer_client.make_directory
-        if host = opts[:host] || @host
-          user = opts[:user] || @user || ENV['USER']
-          transfer_client.set_sftp(user, host)
-        end
-        transfer_client
+        @setup_server = true
+        @directory = File.expand_path(@directory)
+        FileUtils.mkdir_p(@directory)
+        @host = opts[:host] || @host
+        @user = opts[:user] || @user || ENV['USER']
+        true
+      end
+
+      def get_client(same_host)
+        @setup_server ? DRbQS::Transfer::Client.new(@directory, same_host, @host, @user) : nil
       end
     end
   end

data/lib/drbqs/setting/base.rb

@@ -47,6 +47,21 @@ module DRbQS
         @source.value
       end
 
+      def clone
+        new_obj = self.class.new
+        instance_variables.each do |var_name|
+          var = instance_variable_get(var_name)
+          case var
+          when NilClass, FalseClass, TrueClass, Symbol
+            new_var = var
+          else
+            new_var = var.clone
+          end
+          new_obj.instance_variable_set(var_name, new_var)
+        end
+        new_obj
+      end
+
       def preprocess!
       end
       private :preprocess!

data/lib/drbqs/setting/data_container.rb

@@ -30,7 +30,7 @@ module DRbQS
         cl = DRbQS::Setting::Source::DataContainer.new(Array)
         cl.argument = obj.argument.clone
         obj.__data__.each do |key, val|
-          cl.__data__[key] = val
+          cl.__data__[key] = val.clone
         end
         cl
       end

data/lib/drbqs/setting/execute.rb

@@ -25,11 +25,11 @@ module DRbQS
         @port = get_first(:port) do |val|
           val.to_i
         end
-        @no_server = get_first(:no_server)
+        @no_server = set?(:no_server)
         @server = get_first(:server) do |val|
           val.intern
         end
-        @no_node = get_first(:no_node)
+        @no_node = set?(:no_node)
         @node = get_first(:node) do |val|
           val.split(',').map do |s|
             s.intern
@@ -43,7 +43,7 @@ module DRbQS
         else
           @mode = nil
         end
-        if !@output_help && !@definition
+        if @mode != :help && !@definition
           raise DRbQS::Setting::InvalidArgument, "Definition file must be specified."
         end
       end

data/lib/drbqs/setting/node.rb

@@ -74,7 +74,7 @@ module DRbQS
           io.puts "Execute #{@process_num} processes"
         end
 
-        exec_node = DRbQS::ExecuteNode.new(@uri, @options[:log_prefix], @options[:log_level], @options[:node_opts])
+        exec_node = DRbQS::Execution::ExecuteNode.new(@uri, @options[:log_prefix], @options[:log_level], @options[:node_opts])
         exec_node.execute(@process_num)
         exec_node.wait
         true

data/lib/drbqs/setting/server.rb

@@ -131,7 +131,7 @@ module DRbQS
         unless IO === @options[:log_file]
           node_log_file = FileName.create(@options[:log_file], :add => :always, :position => :middle, :delimiter => '', :format => "_node_%02d")
         end
-        exec_node = DRbQS::ExecuteNode.new(uri, node_log_file, @options[:log_level])
+        exec_node = DRbQS::Execution::ExecuteNode.new(uri, node_log_file, @options[:log_level])
         exec_node.execute(@execute_node_number, NODE_INTERVAL_TIME)
         exec_node.wait
       end
@@ -165,7 +165,7 @@ module DRbQS
               io.print "\n" << mes
             end
           rescue => err
-            new_err = err.class.new("Error in loading #{path}: #{err.to_s}")
+            new_err = err.class.new("Error in loading: #{err.to_s}")
             new_err.set_backtrace(err.backtrace)
             raise new_err
           end

data/lib/drbqs/task/registrar.rb

@@ -0,0 +1,39 @@
+module DRbQS
+  class Task
+    # The object of this class is mainly used in DRbQS::Task::Generator#set
+    # and calls Fiber.yield.
+    class Registrar
+      # @param [Hash] data Set instance variables from the hash.
+      def initialize(data)
+        data.each do |key, val|
+          instance_variable_set("@#{key.to_s}", val)
+        end
+      end
+
+      # Add tasks to server.
+      # @param [DRbQS::Task] arg Add an ojbect of DRbQS::Task.
+      # @param [Array] arg Add all elements of an array of objects of DRbQS::Task.
+      def add(arg)
+        case arg
+        when DRbQS::Task
+          Fiber.yield(arg)
+        when Array
+          arg.each { |t| Fiber.yield(t) }
+        else
+          raise ArgumentError, "An argument must be DRbQS::Task or an array of DRbQS::Task."
+        end
+      end
+
+      # Create an object of DRbQS::Task and add it.
+      # The arguments are same as {DRbQS::Task}.
+      def create_add(*args, &block)
+        add(DRbQS::Task.new(*args, &block))
+      end
+
+      # Wait finishes of all tasks in queue of a server.
+      def wait
+        Fiber.yield(:wait)
+      end
+    end
+  end
+end

data/lib/drbqs/task/task.rb

@@ -1,100 +1,180 @@
-require 'drbqs/utility/transfer/file_transfer'
+require 'drbqs/utility/transfer/transfer_file_list'
+require 'drbqs/utility/transfer/transfer'
+require 'drbqs/task/registrar'
 require 'drbqs/task/task_generator'
 
 module DRbQS
 
   # The tasks defined by this class are sent to nodes and
   # calculated by the nodes.
-  # After the node returns the result of calculation to server,
-  # the server execute the hook.
   class Task
+    attr_reader :obj
+    attr_reader :args
+    attr_reader :method_name
     attr_reader :hook
+    attr_accessor :note
 
-    # Nodes execute obj.method_sym(*args).
-    # Server executes &hook with a server instance and an object of result
-    # after the server accepts the results from nodes.
-    def initialize(obj, method_sym, args = [], &hook)
+    # Nodes calculate by obj.method_name(*opts[:args]) and send the result to their server.
+    # Then the server executes &hook with a server instance and an object of result.
+    # For the communication of a server and nodes we must convert obj to a string
+    # by Marshal.dump.
+    # If we set both opts[:hook] and &hook then &hook is prior to opts[:hook].
+    # @param [Object] obj An object that has a method "method_name"
+    # @param [Symbol] method_name Method name of calculation
+    # @param [Hash] opts The options of tasks.
+    # @option opts [Array] :args An array of arguments of method "method_name"
+    # @option opts [String] :note Note for a task
+    # @option opts [Symbol] :hook Method name for hook
+    #  that takes two arguments server and the result object.
+    # @param [Proc] hook A server execute hook as a callback when the server receive the result
+    # @note Changes of obj on a node are not sent to a server.
+    #  That is, opts[:hook] must not depend on changes of instance variables on a node.
+    def initialize(obj, method_name, opts = {}, &hook)
+      @obj = obj
       begin
-        @marshal_obj = Marshal.dump(obj)
+        @marshal_obj = Marshal.dump(@obj)
       rescue
-        raise "Can not dump an instance of #{obj.class}."
+        raise "Can not dump #{@obj.inspect}."
       end
-      unless Array === args
+      @method_name = method_name.intern
+      @args = opts[:args] || []
+      unless Array === @args
         raise "Arguments of task must be an array."
       end
-      @method_sym = method_sym.intern
-      @args = args
-      @hook = hook
+      begin
+        @marshal_args = Marshal.dump(@args)
+      rescue
+        raise "Can not dump #{@args.inspect}."
+      end
+      @note = opts[:note]
+      @hook = hook || opts[:hook]
     end
 
     def drb_args(task_id)
-      [task_id, @marshal_obj, @method_sym, @args]
-    end
-
-    def same_target?(other)
-      @marshal_obj == other.instance_variable_get(:@marshal_obj) &&
-        @method_sym == other.instance_variable_get(:@method_sym) &&
-        @args == other.instance_variable_get(:@args)
+      [task_id, @marshal_obj, @method_name, @marshal_args]
     end
 
     def exec_hook(server, result)
-      if @hook
+      case @hook
+      when Proc
         @hook.call(server, result)
-        true
+      when Symbol, String
+        @obj.__send__(@hook, server, result)
       else
-        nil
+        return nil
       end
+      true
     end
 
-    def self.execute_task(marshal_obj, method_sym, args)
-      obj = Marshal.load(marshal_obj)
-      obj.__send__(method_sym, *args)
-    end
-  end
-
-  # Class to group a number of objects to process tasks.
-  class TaskContainer
-    def initialize(task_ary)
-      @data = task_ary.map.with_index do |task, i|
-        task.drb_args(i)
+    def ==(other)
+      if @marshal_obj == other.instance_variable_get(:@marshal_obj) &&
+          @method_name == other.instance_variable_get(:@method_name) &&
+          @marshal_args == other.instance_variable_get(:@marshal_args)
+        if Proc === @hook && Proc === other.hook
+          # Return false at this time.
+          false
+        else
+          @hook == other.hook
+        end
+      else
+        false
       end
     end
 
-    def exec
-      @data.map do |ary|
-        DRbQS::Task.execute_task(*ary[1..-1])
-      end
+    def self.call_task_method(obj, method_name, args)
+      obj.__send__(method_name, *args)
     end
 
-    def data
-      @data
+    def self.execute_task(marshal_obj, method_name, marshal_args)
+      self.call_task_method(Marshal.load(marshal_obj), method_name, Marshal.load(marshal_args))
     end
-    protected :data
 
-    def ==(other)
-      other.data.each_with_index.all? do |ary, i|
-        ary == @data[i]
+    # DRbQS::Task::TaskSet is a child class of DRbQS::Task and consists of group of a number of tasks.
+    # Objects of the class are generated when we set the option :collect to {DRbQS::Task::Generator#set}
+    # and therefore we are unaware of the objects of DRbQS::TaskSet in many cases.
+    class TaskSet < Task
+
+      class Container
+        def data
+          @data
+        end
+        protected :data
+
+        def ==(other)
+          other.data.each_with_index.all? do |ary, i|
+            ary == @data[i]
+          end
+        end
       end
-    end
-  end
 
-  # Task to group a number of tasks,
-  # which uses TaskContainer and manages hooks of the tasks.
-  class TaskSet < Task
-    def initialize(task_ary)
-      @original_hook = task_ary.map do |task|
-        task.hook
+      class ContainerTask < DRbQS::Task::TaskSet::Container
+        def initialize(task_ary)
+          @data = task_ary
+        end
+
+        def exec
+          @data.map do |task|
+            DRbQS::Task.call_task_method(task.obj, task.method_name, task.args)
+          end
+        end
+
+        def exec_all_hooks(srv, result)
+          result.each_with_index do |res, i|
+            @data[i].exec_hook(srv, res)
+          end
+        end
       end
-      super(DRbQS::TaskContainer.new(task_ary), :exec) do |srv, result|
-        result.each_with_index do |res, i|
-          if hook = @original_hook[i]
-            hook.call(srv, res)
+
+      # Class to group a number of objects to process tasks.
+      class ContainerWithoutHook < DRbQS::Task::TaskSet::Container
+        def initialize(task_ary)
+          @data = task_ary.map.with_index do |task, i|
+            task.drb_args(i)
+          end
+        end
+
+        def exec
+          @data.map do |ary|
+            DRbQS::Task.execute_task(*ary[1..-1])
           end
         end
       end
+
+      attr_reader :original_note
+
+      def initialize(task_ary)
+        @original_note = task_ary.map do |task|
+          task.note
+        end.compact!
+        if task_ary.all? { |task| !(Proc === task.hook) }
+          container = DRbQS::Task::TaskSet::ContainerTask.new(task_ary)
+          super(container, :exec, hook: :exec_all_hooks, note: note_string)
+        else
+          container = DRbQS::Task::TaskSet::ContainerWithoutHook.new(task_ary)
+          @original_task = task_ary
+          super(container, :exec, note: note_string) do |srv, result|
+            result.each_with_index do |res, i|
+              @original_task[i].exec_hook(srv, res)
+            end
+          end
+        end
+      end
+
+      def note_string
+        str = "TaskSet"
+        unless @original_note.empty?
+          case @original_note.size
+          when 1
+            str << ": " << @original_note[0]
+          when 2
+            str << ": " << @original_note.join(", ")
+          else
+            str << ": " << @original_note[0] << ' - ' << @original_note[-1]
+          end
+        end
+        str
+      end
+      private :note_string
     end
   end
-
 end
-
-require 'drbqs/task/command_task'