drbqs 0.0.15 → 0.0.16

data/lib/drbqs/execute/execute_node.rb

@@ -0,0 +1,50 @@
+module DRbQS
+  class Execution
+    class ExecuteNode
+      attr_reader :pid
+
+      def initialize(uri, log_prefix, log_level, node_opts = {})
+        @uri = uri
+        @log_level = log_level
+        if log_prefix
+          @fname = FileName.new(log_prefix, :position => :suffix, :type => :time,
+                                :add => :always, :directory => :parent,
+                                :format => lambda { |t| t.strftime("%Y%m%d_%H%M_#{Process.pid}.log") })
+        else
+          @fname = nil
+        end
+        @pid = []
+        @node_opts = node_opts
+      end
+
+      def get_log_file
+        if @fname
+          return @fname.create
+        end
+        return STDOUT
+      end
+      private :get_log_file
+
+      def create_process
+        @pid << fork do
+          opts = @node_opts.merge({ :log_level => @log_level, :log_file => get_log_file })
+          node = DRbQS::Node.new(@uri, opts)
+          node.connect
+          node.calculate
+        end
+      end
+      private :create_process
+
+      def execute(process_num, interval = 0)
+        process_num.times do |i|
+          create_process
+          sleep(interval) if interval > 0
+        end
+      end
+
+      def wait
+        Process.waitall
+      end
+    end
+  end
+end

data/lib/drbqs/execute/process_define.rb

@@ -2,11 +2,6 @@ require 'drbqs/execute/register'
 
 module DRbQS
   class ProcessDefinition
-    class InvalidServerDefinition < StandardError
-    end
-
-    class InvalidNodeDefinition < StandardError
-    end
 
     attr_reader :register
 
@@ -34,13 +29,19 @@ module DRbQS
     end
 
     def get_server_setting(name = nil)
-      if data = (name ? @register.__server__.assoc(name.intern) : @register.__server__[0])
-        { :name => data[0], :type => data[1][:type], :setting => data[1][:setting], :hostname => data[1][:args][0] }
-      elsif name
-        get_server_setting(nil)
-      else
-        nil
+      if !name
+        data = nil
+        @register.__server__.each do |server_data|
+          unless server_data[1][:template]
+            data = server_data
+            break
+          end
+        end
+        return nil unless data
+      elsif !(data = @register.__server__.assoc(name.intern))
+        return get_server_setting(nil)
       end
+      data
     end
     private :get_server_setting
 
@@ -54,16 +55,33 @@ module DRbQS
     private :get_node_data
 
     def each_node(names = nil, &block)
-      if names
-        names.each do |name|
-          if data = get_node_data(name)
-            yield(name, data)
+      if block_given?
+        if names
+          node_data = []
+          i = 0
+          while i < names.size
+            name = names[i]
+            if data = get_node_data(name)
+              if data[:template]
+                if :group == data[:type]
+                  data[:args].each do |n|
+                    names << n unless names.include?(n)
+                  end
+                end
+              else
+                node_data << [name, data]
+              end
+            end
+            i += 1
+          end
+          node_data.each do |data|
+            yield(*data)
           end
+        else
+          each_node(@register.__node__.map { |name, data| name }, &block)
         end
       else
-        @register.__node__.each do |name, data|
-          yield(name, data)
-        end
+        to_enum(:each_node, names)
       end
     end
     private :each_node
@@ -79,20 +97,24 @@ module DRbQS
     private :server_port
 
     def server_uri(name)
-      data = get_server_setting(name)
-      DRbQS::Misc.create_uri(:host => data[:hostname], :port => server_port)
+      if ary = get_server_setting(name)
+        DRbQS::Misc.create_uri(:host => ary[1][:args][0], :port => server_port)
+      else
+        nil
+      end
     end
     private :server_uri
 
     def execute_server(server_args)
-      if data = get_server_setting(@server)
-        name = data[:name].to_s
-        puts_progress "Execute server '#{name}' (#{data[:type]})"
+      if ary = get_server_setting(@server)
+        name = ary[0].to_s
+        data = ary[1]
+        puts_progress "Execute server '#{name}' (#{data[:ssh] ? 'ssh' : 'local'})"
         setting = data[:setting]
-        hostname = data[:hostname]
+        hostname = data[:args][0]
         type = data[:type]
-        if type == :ssh
-          setting.value.connect name
+        if data[:ssh]
+          setting.value.connect name unless setting.set?(:connect)
           server_setting = setting.mode_setting
         else
           server_setting = setting
@@ -104,7 +126,7 @@ module DRbQS
           server_setting.value.sftp_host hostname
         end
         setting.parse!
-        unless type == :ssh
+        unless data[:ssh]
           server_setting.value.argument.each do |path|
             unless File.exist?(path)
               raise "File '#{path}' does not exist."
@@ -115,20 +137,23 @@ module DRbQS
       end
     rescue Exception => err
       puts_progress "Fail to execute server '#{data[:name].to_s}'"
-      mes = "#{err.to_s} (#{err.class.to_s})"
-      mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
-      new_err = InvalidServerDefinition.new(mes)
+      mes = "Invalid server definition: #{err.to_s} (#{err.class.to_s})"
+      begin
+        mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
+      rescue
+      end
+      new_err = err.class.new(mes)
       new_err.set_backtrace(err.backtrace)
       raise new_err
     end
 
     def execute_one_node(name, data, uri)
-      puts_progress "Execute node '#{name}' (#{data[:type]})"
+      puts_progress "Execute node '#{name}' (#{data[:ssh] ? 'ssh' : 'local'})"
       setting = data[:setting]
-      node_setting = (data[:type] == :ssh ? setting.mode_setting : setting)
+      node_setting = (data[:ssh] ? setting.mode_setting : setting)
       node_setting.value.argument.clear
       node_setting.value.connect uri
-      if data[:type] == :ssh
+      if data[:ssh]
         unless setting.set?(:connect)
           setting.value.connect name.to_s
         end
@@ -142,17 +167,25 @@ module DRbQS
       setting.exec
     rescue Exception => err
       puts_progress "Fail to execute node '#{name.to_s}'"
-      mes = "#{err.to_s} (#{err.class.to_s})"
-      mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
-      new_err = InvalidNodeDefinition.new(mes)
+      mes = "Invalid node definition: #{err.to_s} (#{err.class.to_s})"
+      begin
+        mes = "#{setting.string_for_shell}; " << mes if setting.respond_to?(:string_for_shell)
+      rescue
+      end
+      new_err = err.class.new(mes)
       new_err.set_backtrace(err.backtrace)
       raise new_err
     end
     private :execute_one_node
 
+    def each_node_to_execute(&block)
+      each_node(@node || @register.__default__[:node], &block)
+    end
+    private :each_node_to_execute
+
     def execute_node
       uri = server_uri(@server)
-      each_node(@node) do |name, data|
+      each_node_to_execute do |name, data|
         execute_one_node(name, data, uri)
       end
     end
@@ -160,37 +193,52 @@ module DRbQS
     def information
       info = {}
       info[:server] = @register.__server__.map do |name, data|
-        [name, data[:type]]
+        new_data = data.dup
+        new_data.delete(:setting)
+        [name, new_data]
       end
       info[:node] = @register.__node__.map do |name, data|
-        [name, data[:type]]
+        new_data = data.dup
+        new_data.delete(:setting)
+        [name, new_data]
       end
-      if data = get_server_setting(@server)
-        default_server = data[:name]
+      if ary = get_server_setting(@server)
+        default_server = ary[0]
       else
         default_server = nil
       end
-      info[:default] = {
-        :server => default_server,
-        :node => @node || info[:node].map { |ary| ary[0]},
-        :port => server_port
-      }
+      default_nodes = each_node_to_execute.map do |node_name, node_data|
+        node_name
+      end
+      info[:default] = { :server => default_server, :node => default_nodes, :port => server_port }
       info
     end
 
     def information_string
       info = information
       str = "Server:\n"
-      info[:server].each do |name, type|
-        s = sprintf("%- 16s  %s\n", name, (type == :ssh ? 'ssh' : 'local'))
-        str << (info[:default][:server] == name ? "* " : "  ")
-        str << s
+      ary = (info[:server] + info[:node]).map do |name, data|
+        name.size
+      end
+      string_name_size = ary.max
+      info[:server].each do |name, data|
+        prop = (data[:ssh] ? 'ssh' : 'local')
+        prop << ',template' if data[:template]
+        str << (data[:template] ? " - " : (info[:default][:server] == name ? " * " : "   "))
+        str << sprintf("%- #{string_name_size}s  %s\n", name, prop)
       end
       str << "Node:\n"
-      info[:node].each do |name, type|
-        s = sprintf("%- 16s  %s\n", name, (type == :ssh ? 'ssh' : 'local'))
-        str << (info[:default][:node].include?(name) ? "  " : "- ")
-        str << s
+      info[:node].each do |name, data|
+        if data[:type] == :group
+          prop = 'group: ' << data[:args].map(&:to_s).join(',')
+        else
+          prop = (data[:ssh] ? 'ssh' : 'local')
+          if data[:template]
+            prop << ',template'
+          end
+        end
+        str << (data[:template] ? " - " : (info[:default][:node].include?(name) ? " * " : "   "))
+        str << sprintf("%- #{string_name_size}s  %s\n", name, prop)
       end
       str << "Port: #{info[:default][:port]}"
     end

data/lib/drbqs/execute/register.rb

@@ -1,48 +1,86 @@
 module DRbQS
   class ProcessDefinition
     class Register
-      attr_reader :__server__, :__node__, :__default__, :__usage__
+      # @return [Array] an array of pair [name symbol, definition hash]
+      # Keys of the hash are :type, :template, :ssh, :setting, and :args.
+      attr_reader :__server__, :__node__
+
+      # @return [Array] a hash of key and value
+      attr_reader :__default__, :__usage__
 
       def initialize
         @__server__ = []
         @__node__ = []
         @__default__ = {}
-        @__usage__ = nil
+        @__usage__ = {}
       end
 
-      def __register__(type, name, *args, &block)
-        if block_given?
-          type = type.intern
-          case type
-          when :server
-            setting = DRbQS::Setting::Server.new
-          when :node
-            setting = DRbQS::Setting::Node.new
+      def __register__(type, setting, name, template, args, &block)
+        ary = [name.intern, { :type => type.intern, :template => template, :args => args }]
+        type = type.to_s
+        case block.arity
+        when 2
+          if DRbQS::Setting::SSH === setting
+            ssh_setting = setting
           else
-            raise ArgumentError, "Invalid type of setting '#{type.to_s}'"
-          end
-          case block.arity
-          when 2
             ssh_setting = DRbQS::Setting::SSH.new
             ssh_setting.value.argument << type.to_s
-            yield(setting.value, ssh_setting.value)
             ssh_setting.mode_setting = setting
-            unless ssh_setting.value.argument[0] == type.to_s
-              ssh_setting.value.argument.unshift(type.to_s)
+          end
+          yield(ssh_setting.mode_setting.value, ssh_setting.value)
+          unless ssh_setting.value.argument[0] == type
+            ssh_setting.value.argument.unshift(type)
+          end
+          ary[1][:ssh] = true
+          ary[1][:setting] = ssh_setting
+        when 1
+          if DRbQS::Setting::SSH === setting
+            raise ArgumentError, "Inherited definition is over ssh."
+          end
+          yield(setting.value)
+          ary[1][:ssh] = false
+          ary[1][:setting] = setting
+        else
+          raise ArgumentError, "Block must take one or two arguments."
+        end
+        ary
+      end
+      private :__register__
+
+      def __register_server__(template, name, load_def, *args, &block)
+        if load_def
+          if DRbQS::Setting::Base === load_def
+            setting = load_def
+          elsif data = @__server__.assoc(load_def.intern)
+            setting = data[1][:setting].clone
+          else
+            raise ArgumentError, "Not registered definition '#{load_def}'."
+          end
+        else
+          setting = DRbQS::Setting::Server.new
+        end
+        @__server__ << __register__(:server, setting, name, template, args, &block)
+      end
+      private :__register_server__
+
+      def __register_node__(template, name, load_def, *args, &block)
+        if load_def
+          if DRbQS::Setting::Base === load_def
+            setting = load_def
+          elsif data = @__node__.assoc(load_def.intern)
+            if data[1][:type] == :group
+              raise ArgumentError, "Definition to inherit is group."
             end
-            ary = [name.intern, { :type => :ssh, :setting => ssh_setting, :args => args }]
-          when 1
-            yield(setting.value)
-            ary = [name.intern, { :type => type, :setting => setting, :args => args }]
+            setting = data[1][:setting].clone
           else
-            raise ArgumentError, "Block must take one or two arguments."
+            raise ArgumentError, "Not registered definition '#{load_def}'."
           end
-          instance_variable_get("@__#{type.to_s}__") << ary
         else
-          raise ArgumentError, "Block to define settings is not given."
+          setting = DRbQS::Setting::Node.new
         end
+        @__node__ << __register__(:node, setting, name, template, args, &block)
       end
-      private :__register__
+      private :__register_node__
 
       # To set properties of server we can use the similar options to the command 'drbqs-server'.
       # When we execute a server over ssh, we can use the similar options to the command 'drbqs-ssh'
@@ -51,34 +89,80 @@ module DRbQS
       # If we omit the 'connect' method then the program tries to connect
       # the name specified as first argument.
       # 
-      # * Example of a server on localhost
-      # register_server(:server_local, "example.com") do |server|
-      #   server.load "server_definition.rb"
-      #   server.acl "/path/to/acl"
-      #   server.log_file "/path/to/log"
-      #   server.log_level Logger::ERROR
-      #   server.sftp_user "username"
-      #   server.sftp_host "example.com"
-      # end
+      # @param [Symbol,String] name Server name
+      # @param [Hash] opts The options of server
+      # @option opts [true,false] :template Template for other servers to load, not actual server
+      # @option opts [Symbol] :load Inherit definition of other server
       # 
-      # * Example of a server over ssh
-      # register_server(:server_ssh, "example.co.jp") do |server, ssh|
-      #   server.load "server_definition.rb"
-      #   server.acl "/path/to/acl"
-      #   server.log_level Logger::ERROR
-      #   server.sftp_user "username"
-      #   server.sftp_host "example.com"
+      # @example A server on localhost
+      #  server :server_local, "example.com" do |srv|
+      #    srv.load "server_definition.rb"
+      #    srv.acl "/path/to/acl"
+      #    srv.log_file "/path/to/log"
+      #    srv.log_level Logger::ERROR
+      #    srv.sftp_user "username"
+      #    srv.sftp_host "example.com"
+      #  end
       # 
-      #   ssh.connect "hostname"
-      #   ssh.directory "/path/to/dir"
-      #   ssh.shell "bash"
-      #   ssh.rvm "ruby-head"
-      #   ssh.rvm_init "/path/to/scripts/rvm"
-      #   ssh.output "/path/to/output"
-      #   ssh.nice 10
-      # end
-      def register_server(name, hostname = nil, &block)
-        __register__(:server, name, hostname || name.to_s, &block)
+      # @example A server over ssh
+      #  server :server_ssh, "example.co.jp" do |srv, ssh|
+      #    srv.load "server_definition.rb"
+      #    srv.acl "/path/to/acl"
+      #    srv.log_level Logger::ERROR
+      #    srv.sftp_user "username"
+      #    srv.sftp_host "example.com"
+      #  
+      #    ssh.connect "hostname"
+      #    ssh.directory "/path/to/dir"
+      #    ssh.shell "bash"
+      #    ssh.rvm "ruby-head"
+      #    ssh.rvm_init "/path/to/scripts/rvm"
+      #    ssh.output "/path/to/output"
+      #    ssh.nice 10
+      #  end
+      def server(name, *args, &block)
+        name = name.intern
+        if ind = @__server__.index { |n, data| name == n }
+          old_data = @__server__.delete_at(ind)
+        else
+          old_data = nil
+        end
+        unless block_given?
+          raise ArgumentError, "Block to define settings is not given."
+        end
+        case args.size
+        when 2
+          hostname = args[0]
+          opts = args[1]
+          unless Hash === opts
+            raise ArgumentError, "Options must be hash."
+          end
+        when 1
+          if Hash === args[0]
+            hostname = nil
+            opts = args[0]
+          else
+            hostname = args[0]
+            opts = {}
+          end
+        else
+          unless old_data
+            raise ArgumentError, "Invalid argument size."
+          end
+        end
+        if old_data
+          if opts[:load]
+            raise ArgumentError, "Can not set both reconfiguring and loading."
+          end
+          load_def = old_data[1][:setting]
+          hostname = old_data[1][:args][0] if !hostname
+        else
+          load_def = opts[:load]
+        end
+        if !opts[:template] && !hostname
+          raise ArgumentError, "Definition of server '#{name}' needs hostname."
+        end
+        __register_server__(opts[:template], name, load_def, hostname, &block)
       end
 
       # To set properties of nodes we can use the similar options to the command 'drbqs-node'.
@@ -87,56 +171,126 @@ module DRbQS
       # If we omit the 'connect' method then the program tries to connect
       # the name specified as first argument.
       # 
-      # * Example of nodes on localhost
-      # register_node(:node_local) do |node|
-      #   node.process 3
-      #   node.load "load_lib.rb"
-      #   node.log_prefix "/path/to/log"
-      #   node.log_level Logger::DEBUG
-      # end
+      # @param [Symbol,String] name Node name
+      # @param [Hash] opts The options of node
+      # @option opts [true,false] :template Template for other nodes to load, not actual node
+      # @option opts [Symbol] :load Inherit definition of other node
+      # @option opts [true,false] :group Define the group of node
+      # 
+      # @example Nodes on localhost
+      #  node :node_local do |nd|
+      #    nd.process 3
+      #    nd.load "load_lib.rb"
+      #    nd.log_prefix "/path/to/log"
+      #    nd.log_level Logger::DEBUG
+      #  end
       # 
-      # * Example of nodes over ssh
-      # register_node(:node_ssh) do |node, ssh|
-      #   node.process 3
-      #   node.load "load_lib.rb"
-      #   node.log_level Logger::DEBUG
+      # @example Nodes over ssh
+      #  node :node_ssh do |nd, ssh|
+      #    nd.process 3
+      #    nd.load "load_lib.rb"
+      #    nd.log_level Logger::DEBUG
+      #  
+      #    ssh.connect "hostname"
+      #    ssh.directory "/path/to/dir"
+      #    ssh.shell "bash"
+      #    ssh.rvm "ruby-head"
+      #    ssh.rvm_init "/path/to/scripts/rvm"
+      #    ssh.output "/path/to/output"
+      #    ssh.nice 10
+      #  end
       # 
-      #   ssh.connect "hostname"
-      #   ssh.directory "/path/to/dir"
-      #   ssh.shell "bash"
-      #   ssh.rvm "ruby-head"
-      #   ssh.rvm_init "/path/to/scripts/rvm"
-      #   ssh.output "/path/to/output"
-      #   ssh.nice 10
-      # end
-      def register_node(name, &block)
-        __register__(:node, name, &block)
+      # @example Node group
+      #  node :node_group, :group => [:node_local, :node_ssh]
+      def node(name, opts = {}, &block)
+        name = name.intern
+        load_def = opts[:load]
+        if ind = @__node__.index { |n, data| name == n }
+          old_data = @__node__.delete_at(ind)
+          if (opts[:group] && old_data[1][:type] != :group) ||
+              (!opts[:group] && old_data[1][:type] == :group)
+            raise ArgumentError, "Change type of definition on reconfiguring."
+          elsif (!opts[:group] && load_def)
+            raise ArgumentError, "Can not set both reconfiguring and loading."
+          end
+          load_def = old_data[1][:setting]
+        else
+          old_data = nil
+        end
+        if opts[:group]
+          unless Array === opts[:group]
+            raise ":group must be an array of node names."
+          end
+          data = {
+            :type => :group, :template => true, :ssh => nil, :setting => nil,
+            :args => opts[:group].map(&:intern)
+          }
+          @__node__ << [name, data]
+        elsif block_given?
+          __register_node__(opts[:template], name, load_def, &block)
+        else
+          raise ArgumentError, "Block to define settings is not given."
+        end
+      end
+
+      # @param [Array] *args Symbols of servers
+      # @example Clear server definitions
+      #  clear_server :server1, :server2
+      def clear_server(*args)
+        args.each do |arg|
+          @__server__.delete_if do |name, data|
+            name == arg.intern
+          end
+        end
+      end
+
+      # @param [Array] *args Symbols of nodes
+      # @example Clear node definitions
+      #  clear_node :node1, :node2
+      def clear_node(*args)
+        args.each do |arg|
+          @__node__.delete_if do |name, data|
+            name == arg.intern
+          end
+        end
       end
 
       # We can set default server, default port, and default directory to output log.
-      # * Example of usage
-      #   default :port => 13456, :server => :server_local, :log => "/tmp/drbqs_execute_log"
+      # @param [Hash] val the pair of key and value
+      # @option val [Fixnum] :port Port number of a server
+      # @option val [Symbol] :server Server executed by default
+      # @option val [String] :log Path of log of a server and nods on localhost
+      # @example Set default value
+      #  default :port => 13456, :server => :server_local, :log => "/tmp/drbqs_execute_log"
       def default(val = {})
         val.delete_if { |key, v| !v }
-        @__default__.merge!(val)
-        if @__default__[:server]
-          @__default__[:server] = @__default__[:server].intern
+        if val[:server]
+          val[:server] = val[:server].intern
         end
-        if @__default__[:port]
-          @__default__[:port] = @__default__[:port].to_i
+        if val[:port]
+          val[:port] = val[:port].to_i
         end
+        raise "Invalid default value for :node." if val.has_key?(:node) && !(Array === val[:node])
+        raise "Invalid default value for :log." if val.has_key?(:log) && !(String === val[:log])
+        @__default__.merge!(val)
       end
 
-      def default_clear(key)
-        @__default__.delete(key)
+      # @example Clear default value
+      #  default_clear :port, :server, :log
+      def default_clear(*keys)
+        keys.each do |key|
+          @__default__.delete(key)
+        end
       end
 
-      # We can set messages of usage and path of definition file of server
-      # to output help of server on showing help.
-      # * Example of usage
-      # usage(:message => 'Calculate some value', :server => 'server.rb')
+      # We can set some messages shown by drbqs-execute -h.
+      # @param [Hash] opts
+      # @option opts [String] :message Simple message strings
+      # @option opts [String] :server Path of server definition to output as help of server
+      # @example Set usage
+      #  usage :message => 'Calculate some value', :server => 'server.rb'
       def usage(opts = {})
-        @__usage__ = opts
+        @__usage__.merge!(opts)
       end
 
       def __load__(path)