mcollective-client 2.0.0 → 2.2.0

data/lib/mcollective/rpc/agent.rb

@@ -10,24 +10,17 @@ module MCollective
     # usage would be:
     #
     #    module MCollective
-    #       module Agent
-    #          class Helloworld<RPC::Agent
-    #             matadata :name        => "Test SimpleRPC Agent",
-    #                      :description => "A simple test",
-    #                      :author      => "You",
-    #                      :license     => "1.1",
-    #                      :url         => "http://your.com/,
-    #                      :timeout     => 60
-    #
-    #             action "hello" do
-    #                 reply[:msg] = "Hello #{request[:name]}"
-    #             end
+    #      module Agent
+    #        class Helloworld<RPC::Agent
+    #          action "hello" do
+    #            reply[:msg] = "Hello #{request[:name]}"
+    #          end
     #
-    #             action "foo" do
-    #                 implemented_by "/some/script.sh"
-    #             end
+    #          action "foo" do
+    #            implemented_by "/some/script.sh"
     #          end
-    #       end
+    #        end
+    #      end
     #    end
     #
     # If you wish to implement the logic for an action using an external script use the
@@ -38,32 +31,16 @@ module MCollective
     #
     # We also currently have the validation code in here, this will be moved to plugins soon.
     class Agent
-      attr_accessor :meta, :reply, :request
-      attr_reader :logger, :config, :timeout, :ddl
+      attr_accessor :reply, :request, :agent_name
+      attr_reader :logger, :config, :timeout, :ddl, :meta
 
       def initialize
-        # Default meta data unset
-        @meta = {:timeout     => 10,
-                 :name        => "Unknown",
-                 :description => "Unknown",
-                 :author      => "Unknown",
-                 :license     => "Unknown",
-                 :version     => "Unknown",
-                 :url         => "Unknown"}
-
-        @timeout = meta[:timeout] || 10
-        @logger = Log.instance
-        @config = Config.instance
         @agent_name = self.class.to_s.split("::").last.downcase
 
-        # Loads the DDL so we can later use it for validation
-        # and help generation
-        begin
-          @ddl = DDL.new(@agent_name)
-        rescue Exception => e
-          Log.debug("Failed to load DDL for agent: #{e.class}: #{e}")
-          @ddl = nil
-        end
+        load_ddl
+
+        @logger = Log.instance
+        @config = Config.instance
 
         # if we have a global authorization provider enable it
         # plugins can still override it per plugin
@@ -72,17 +49,32 @@ module MCollective
         startup_hook
       end
 
+      def load_ddl
+        @ddl = DDL.new(@agent_name, :agent)
+        @meta = @ddl.meta
+        @timeout = @meta[:timeout] || 10
+
+      rescue Exception => e
+        Log.error("Failed to load DDL for the '%s' agent, DDLs are required: %s: %s" % [@agent_name, e.class, e.to_s])
+        raise DDLValidationError
+      end
+
       def handlemsg(msg, connection)
-        @request = RPC.request(msg)
-        @reply = RPC.reply
+        @request = RPC::Request.new(msg, @ddl)
+        @reply = RPC::Reply.new(@request.action, @ddl)
 
         begin
+          # Incoming requests need to be validated against the DDL thus reusing
+          # all the work users put into creating DDLs and creating a consistant
+          # quality of input validation everywhere with the a simple once off
+          # investment of writing a DDL
+          @request.validate!
+
           # Calls the authorization plugin if any is defined
           # if this raises an exception we wil just skip processing this
           # message
           authorization_hook(@request) if respond_to?("authorization_hook")
 
-
           # Audits the request, currently continues processing the message
           # we should make this a configurable so that an audit failure means
           # a message wont be processed by this node depending on config
@@ -157,21 +149,6 @@ module MCollective
         return true
       end
 
-      # Generates help using the template based on the data
-      # created with metadata and input
-      def self.help(template)
-        if @ddl
-          @ddl.help(template)
-        else
-          "No DDL defined"
-        end
-      end
-
-      # to auto generate help
-      def help
-        self.help("#{@config[:configdir]}/rpc-help.erb")
-      end
-
       # Returns an array of actions this agent support
       def self.actions
         public_instance_methods.sort.grep(/_action$/).map do |method|
@@ -179,7 +156,6 @@ module MCollective
         end
       end
 
-
       private
       # Runs a command via the MC::Shell wrapper, options are as per MC::Shell
       #
@@ -255,20 +231,7 @@ module MCollective
 
       # Registers meta data for the introspection hash
       def self.metadata(data)
-        [:name, :description, :author, :license, :version, :url, :timeout].each do |arg|
-          raise "Metadata needs a :#{arg}" unless data.include?(arg)
-        end
-
-        # Our old style agents were able to do all sorts of things to the meta
-        # data during startup_hook etc, don't really want that but also want
-        # backward compat.
-        #
-        # Here if you're using the new metadata way this replaces the getter
-        # with one that always return the same data, setter will still work but
-        # wont actually do anything of note.
-        define_method("meta") {
-          data
-        }
+        Log.warn("%s: setting meta data in agents have been deprecated, DDL files are now being used for this information." % File.basename(caller.first))
       end
 
       # Creates the needed activate? class in a manner similar to the other
@@ -327,51 +290,12 @@ module MCollective
       # validate :command, ["start", "stop"]
       #
       # It will raise appropriate exceptions that the RPC system understand
-      #
-      # TODO: this should be plugins, 1 per validatin method so users can add their own
-      #       at the moment i have it here just to proof the point really
       def validate(key, validation)
         raise MissingRPCData, "please supply a #{key} argument" unless @request.include?(key)
 
-        if validation.is_a?(Regexp)
-          raise InvalidRPCData, "#{key} should match #{validation}" unless @request[key].match(validation)
-
-        elsif validation.is_a?(Symbol)
-          case validation
-          when :shellsafe
-            raise InvalidRPCData, "#{key} should be a String" unless @request[key].is_a?(String)
-
-            ['`', '$', ';', '|', '&&', '>', '<'].each do |chr|
-              raise InvalidRPCData, "#{key} should not have #{chr} in it" if @request[key].match(Regexp.escape(chr))
-            end
-
-          when :ipv6address
-            begin
-              require 'ipaddr'
-              ip = IPAddr.new(@request[key])
-              raise InvalidRPCData, "#{key} should be an ipv6 address" unless ip.ipv6?
-            rescue
-              raise InvalidRPCData, "#{key} should be an ipv6 address"
-            end
-
-          when :ipv4address
-            begin
-              require 'ipaddr'
-              ip = IPAddr.new(@request[key])
-              raise InvalidRPCData, "#{key} should be an ipv4 address" unless ip.ipv4?
-            rescue
-              raise InvalidRPCData, "#{key} should be an ipv4 address"
-            end
-
-          when :boolean
-            raise InvalidRPCData, "#{key} should be boolean" unless [TrueClass, FalseClass].include?(@request[key].class)
-          end
-        elsif validation.is_a?(Array)
-          raise InvalidRPCData, "#{key} should be one of %s" % [ validation.join(", ") ] unless validation.include?(@request[key])
-
-        else
-          raise InvalidRPCData, "#{key} should be a #{validation}" unless  @request[key].is_a?(validation)
-        end
+        Validator.validate(@request[key], validation)
+      rescue ValidatorError => e
+        raise InvalidRPCData, "Input %s did not pass validation: %s" % [ key, e.message ]
       end
 
       # convenience wrapper around Util#shellescape

data/lib/mcollective/rpc/client.rb

@@ -3,8 +3,9 @@ module MCollective
     # The main component of the Simple RPC client system, this wraps around MCollective::Client
     # and just brings in a lot of convention and standard approached.
     class Client
-      attr_accessor :discovery_timeout, :timeout, :verbose, :filter, :config, :progress, :ttl, :reply_to
+      attr_accessor :timeout, :verbose, :filter, :config, :progress, :ttl, :reply_to
       attr_reader :client, :stats, :ddl, :agent, :limit_targets, :limit_method, :output_format, :batch_size, :batch_sleep_time, :batch_mode
+      attr_reader :discovery_options, :discovery_method, :limit_seed
 
       @@initial_options = nil
 
@@ -37,20 +38,24 @@ module MCollective
           @@initial_options = Marshal.dump(initial_options)
         end
 
+        @initial_options = initial_options
         @stats = Stats.new
         @agent = agent
-        @discovery_timeout = initial_options[:disctimeout]
-        @timeout = initial_options[:timeout]
+        @timeout = initial_options[:timeout] || 5
         @verbose = initial_options[:verbose]
-        @filter = initial_options[:filter]
+        @filter = initial_options[:filter] || Util.empty_filter
         @config = initial_options[:config]
         @discovered_agents = nil
         @progress = initial_options[:progress_bar]
         @limit_targets = initial_options[:mcollective_limit_targets]
         @limit_method = Config.instance.rpclimitmethod
+        @limit_seed = initial_options[:limit_seed] || nil
         @output_format = initial_options[:output_format] || :console
         @force_direct_request = false
         @reply_to = initial_options[:reply_to]
+        @discovery_method = initial_options[:discovery_method] || Config.instance.default_discovery_method
+        @discovery_options = initial_options[:discovery_options] || []
+        @force_display_mode = initial_options[:force_display_mode] || false
 
         @batch_size = Integer(initial_options[:batch_size] || 0)
         @batch_sleep_time = Float(initial_options[:batch_sleep_time] || 1)
@@ -61,6 +66,8 @@ module MCollective
         @client = MCollective::Client.new(@config)
         @client.options = initial_options
 
+        @discovery_timeout = discovery_timeout
+
         @collective = @client.collective
         @ttl = initial_options[:ttl] || Config.instance.ttl
 
@@ -77,13 +84,11 @@ module MCollective
         # We do this only if the timeout is the default 5
         # seconds, so that users cli overrides will still
         # get applied
-        begin
-          @ddl = DDL.new(agent)
-          @timeout = @ddl.meta[:timeout] + @discovery_timeout if @timeout == 5
-        rescue Exception => e
-          Log.debug("Could not find DDL: #{e}")
-          @ddl = nil
-        end
+        #
+        # DDLs are required, failure to find a DDL is fatal
+        @ddl = DDL.new(agent)
+        @stats.ddl = @ddl
+        @timeout = @ddl.meta[:timeout] + @discovery_timeout if @timeout == 5
 
         # allows stderr and stdout to be overridden for testing
         # but also for web apps that might not want a bunch of stuff
@@ -110,11 +115,7 @@ module MCollective
 
       # Returns help for an agent if a DDL was found
       def help(template)
-        if @ddl
-          @ddl.help(template)
-        else
-          return "Can't find DDL for agent '#{@agent}'"
-        end
+        @ddl.help(template)
       end
 
       # Creates a suitable request hash for the SimpleRPC agent.
@@ -215,7 +216,7 @@ module MCollective
 
         @stats.reset
 
-        @ddl.validate_request(action, args) if @ddl
+        @ddl.validate_rpc_request(action, args) if @ddl
 
         # if a global batch size is set just use that else set it
         # in the case that it was passed as an argument
@@ -272,7 +273,7 @@ module MCollective
       # request which technically does not need filters.  If you try to use this
       # mode with direct addressing disabled an exception will be raise
       def custom_request(action, args, expected_agents, filter = {}, &block)
-        @ddl.validate_request(action, args) if @ddl
+        @ddl.validate_rpc_request(action, args) if @ddl
 
         if filter == {} && !Config.instance.direct_addressing
           raise "Attempted to do a filterless custom_request without direct_addressing enabled, preventing unexpected call to all nodes"
@@ -319,6 +320,30 @@ module MCollective
         end
       end
 
+      def discovery_timeout
+        return @initial_options[:disctimeout] if @initial_options[:disctimeout]
+        return @client.discoverer.ddl.meta[:timeout]
+      end
+
+      def discovery_method=(method)
+        @discovery_method = method
+
+        if @initial_options[:discovery_options]
+          @discovery_options = @initial_options[:discovery_options]
+        else
+          @discovery_options.clear
+        end
+
+        @client.options = options
+        @discovery_timeout = discovery_timeout
+        reset
+      end
+
+      def discovery_options=(options)
+        @discovery_options = [options].flatten
+        reset
+      end
+
       # Sets the class filter
       def class_filter(klass)
         @filter["cf_class"] << klass
@@ -359,7 +384,7 @@ module MCollective
 
       # Set a compound filter
       def compound_filter(filter)
-        @filter["compound"] << Matcher::Parser.new(filter).execution_stack
+        @filter["compound"] <<  Matcher.create_compound_callstack(filter)
         reset
       end
 
@@ -431,8 +456,12 @@ module MCollective
           # only really be for a few -I's on the CLI
           #
           # For safety we leave the filter in place for now, that way we can support this
-          # enhancement also in broadcast mode
-          elsif options[:filter]["identity"].size > 0
+          # enhancement also in broadcast mode.
+          #
+          # This is only needed for the 'mc' discovery method, other methods might change
+          # the concept of identity to mean something else so we should pass the full
+          # identity filter to them
+          elsif options[:filter]["identity"].size > 0 && @discovery_method == "mc"
             regex_filters = options[:filter]["identity"].select{|i| i.match("^\/")}.size
 
             if regex_filters == 0
@@ -446,21 +475,39 @@ module MCollective
         unless @discovered_agents
           @stats.time_discovery :start
 
-          @stderr.print("Determining the amount of hosts matching filter for #{discovery_timeout} seconds .... ") if verbose
+          @client.options = options
+
+          # if compound filters are used the only real option is to use the mc
+          # discovery plugin since its the only capable of using data queries etc
+          # and we do not want to degrade that experience just to allow compounds
+          # on other discovery plugins the UX would be too bad raising complex sets
+          # of errors etc.
+          @client.discoverer.force_discovery_method_by_filter(options[:filter])
+
+          if verbose
+            actual_timeout = @client.discoverer.discovery_timeout(discovery_timeout, options[:filter])
+
+            if actual_timeout > 0
+              @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [@client.discoverer.discovery_method, actual_timeout])
+            else
+              @stderr.print("Discovering hosts using the %s method .... " % [@client.discoverer.discovery_method])
+            end
+          end
 
           # if the requested limit is a pure number and not a percent
           # and if we're configured to use the first found hosts as the
           # limit method then pass in the limit thus minimizing the amount
           # of work we do in the discover phase and speeding it up significantly
           if @limit_method == :first and @limit_targets.is_a?(Fixnum)
-            @discovered_agents = @client.discover(@filter, @discovery_timeout, @limit_targets)
+            @discovered_agents = @client.discover(@filter, discovery_timeout, @limit_targets)
           else
-            @discovered_agents = @client.discover(@filter, @discovery_timeout)
+            @discovered_agents = @client.discover(@filter, discovery_timeout)
           end
 
-          @force_direct_request = false
           @stderr.puts(@discovered_agents.size) if verbose
 
+          @force_direct_request = @client.discoverer.force_direct_mode?
+
           @stats.time_discovery :end
         end
 
@@ -480,13 +527,19 @@ module MCollective
          :collective => @collective,
          :output_format => @output_format,
          :ttl => @ttl,
+         :discovery_method => @discovery_method,
+         :discovery_options => @discovery_options,
+         :force_display_mode => @force_display_mode,
          :config => @config}
       end
 
       # Sets the collective we are communicating with
       def collective=(c)
+        raise "Unknown collective #{c}" unless Config.instance.collectives.include?(c)
+
         @collective = c
-        @client.options[:collective] = c
+        @client.options = options
+        reset
       end
 
       # Sets and sanity checks the limit_targets variable
@@ -529,7 +582,6 @@ module MCollective
         @batch_sleep_time = Float(time)
       end
 
-      private
       # Pick a number of nodes from the discovered nodes
       #
       # The count should be a string that can be either
@@ -542,12 +594,14 @@ module MCollective
       #   - :first would be a simple way to do a distance based
       #     selection
       #   - anything else will just pick one at random
+      #   - if random chosen, and batch-seed set, then set srand
+      #     for the generator, and reset afterwards
       def pick_nodes_from_discovered(count)
         if count =~ /%$/
-          pct = (discover.size * (count.to_f / 100)).to_i
+          pct = Integer((discover.size * (count.to_f / 100)))
           pct == 0 ? count = 1 : count = pct
         else
-          count = count.to_i
+          count = Integer(count)
         end
 
         return discover if discover.size <= count
@@ -557,16 +611,58 @@ module MCollective
         if @limit_method == :first
           return discover[0, count]
         else
+          # we delete from the discovered list because we want
+          # to be sure there is no chance that the same node will
+          # be randomly picked twice.  So we have to clone the
+          # discovered list else this method will only ever work
+          # once per discovery cycle and not actually return the
+          # right nodes.
+          haystack = discover.clone
+
+          if @limit_seed
+            haystack.sort!
+            srand(@limit_seed)
+          end
+
           count.times do
-            rnd = rand(discover.size)
-            result << discover[rnd]
-            discover.delete_at(rnd)
+            rnd = rand(haystack.size)
+            result << haystack.delete_at(rnd)
           end
+
+          # Reset random number generator to fresh seed
+          # As our seed from options is most likely short
+          srand if @limit_seed
         end
 
         [result].flatten
       end
 
+      def load_aggregate_functions(action, ddl)
+        return nil unless ddl
+        return nil unless ddl.action_interface(action).keys.include?(:aggregate)
+
+        return Aggregate.new(ddl.action_interface(action))
+
+      rescue => e
+        Log.error("Failed to load aggregate functions, calculating summaries disabled: %s: %s (%s)" % [e.backtrace.first, e.to_s, e.class])
+        return nil
+      end
+
+      def aggregate_reply(reply, aggregate)
+        return nil unless aggregate
+
+        aggregate.call_functions(reply)
+        return aggregate
+      rescue Exception => e
+        Log.error("Failed to calculate aggregate summaries for reply from %s, calculating summaries disabled: %s: %s (%s)" % [reply[:senderid], e.backtrace.first, e.to_s, e.class])
+        return nil
+      end
+
+      def rpc_result_from_reply(agent, action, reply)
+        Result.new(agent, action, {:sender => reply[:senderid], :statuscode => reply[:body][:statuscode],
+                                   :statusmsg => reply[:body][:statusmsg], :data => reply[:body][:data]})
+      end
+
       # for requests that do not care for results just
       # return the request id and don't do any of the
       # response processing.
@@ -576,7 +672,7 @@ module MCollective
       #
       # Should only be called via method_missing
       def fire_and_forget_request(action, args, filter=nil)
-        @ddl.validate_request(action, args) if @ddl
+        @ddl.validate_rpc_request(action, args) if @ddl
 
         req = new_request(action.to_s, args)
 
@@ -607,31 +703,43 @@ module MCollective
         @force_direct_request = true
 
         discovered = discover
-        result = []
+        results = []
         respcount = 0
 
         if discovered.size > 0
           req = new_request(action.to_s, args)
 
+          aggregate = load_aggregate_functions(action, @ddl)
+
           if @progress && !block_given?
             twirl = Progress.new
             @stdout.puts
             @stdout.print twirl.twirl(respcount, discovered.size)
           end
 
+          @stats.requestid = nil
+
           discovered.in_groups_of(batch_size) do |hosts, last_batch|
             message = Message.new(req, nil, {:agent => @agent, :type => :direct_request, :collective => @collective, :filter => opts[:filter], :options => opts})
+
+            # first time round we let the Message object create a request id
+            # we then re-use it for future requests to keep auditing sane etc
+            @stats.requestid = message.create_reqid unless @stats.requestid
+            message.requestid = @stats.requestid
+
             message.discovered_hosts = hosts.clone.compact
 
             @client.req(message) do |resp|
               respcount += 1
 
               if block_given?
-                process_results_with_block(action, resp, block)
+                aggregate = process_results_with_block(action, resp, block, aggregate)
               else
                 @stdout.print twirl.twirl(respcount, discovered.size) if @progress
 
-                result << process_results_without_block(resp, action)
+                result, aggregate = process_results_without_block(resp, action, aggregate)
+
+                results << result
               end
             end
 
@@ -643,6 +751,8 @@ module MCollective
 
             sleep sleep_time unless last_batch
           end
+
+          @stats.aggregate_summary = aggregate.summarize if aggregate
         else
           @stderr.print("\nNo request sent, we did not discover any nodes.")
         end
@@ -656,7 +766,7 @@ module MCollective
         if block_given?
           return stats
         else
-          return [result].flatten
+          return [results].flatten
         end
       end
 
@@ -693,30 +803,36 @@ module MCollective
 
         message = Message.new(req, nil, {:agent => @agent, :type => :request, :collective => @collective, :filter => opts[:filter], :options => opts})
         message.discovered_hosts = discovered.clone
-        message.type = :direct_request if @force_direct_request
 
-        result = []
+        results = []
         respcount = 0
 
         if discovered.size > 0
+          message.type = :direct_request if @force_direct_request
+
           if @progress && !block_given?
             twirl = Progress.new
             @stdout.puts
             @stdout.print twirl.twirl(respcount, discovered.size)
           end
 
+          aggregate = load_aggregate_functions(action, @ddl)
+
           @client.req(message) do |resp|
             respcount += 1
 
             if block_given?
-              process_results_with_block(action, resp, block)
+              aggregate = process_results_with_block(action, resp, block, aggregate)
             else
               @stdout.print twirl.twirl(respcount, discovered.size) if @progress
 
-              result << process_results_without_block(resp, action)
+              result, aggregate = process_results_without_block(resp, action, aggregate)
+
+              results << result
             end
           end
 
+          @stats.aggregate_summary = aggregate.summarize if aggregate
           @stats.client_stats = @client.stats
         else
           @stderr.print("\nNo request sent, we did not discover any nodes.")
@@ -731,62 +847,68 @@ module MCollective
         if block_given?
           return stats
         else
-          return [result].flatten
+          return [results].flatten
         end
       end
 
       # Handles result sets that has no block associated, sets fails and ok
       # in the stats object and return a hash of the response to send to the
       # caller
-      def process_results_without_block(resp, action)
+      def process_results_without_block(resp, action, aggregate)
         @stats.node_responded(resp[:senderid])
 
+        result = rpc_result_from_reply(@agent, action, resp)
+        aggregate = aggregate_reply(result, aggregate) if aggregate
+
         if resp[:body][:statuscode] == 0 || resp[:body][:statuscode] == 1
           @stats.ok if resp[:body][:statuscode] == 0
           @stats.fail if resp[:body][:statuscode] == 1
-
-          return Result.new(@agent, action, {:sender => resp[:senderid], :statuscode => resp[:body][:statuscode],
-                              :statusmsg => resp[:body][:statusmsg], :data => resp[:body][:data]})
         else
           @stats.fail
-
-          return Result.new(@agent, action, {:sender => resp[:senderid], :statuscode => resp[:body][:statuscode],
-                              :statusmsg => resp[:body][:statusmsg], :data => nil})
         end
+
+        [result, aggregate]
       end
 
       # process client requests by calling a block on each result
       # in this mode we do not do anything fancy with the result
       # objects and we raise exceptions if there are problems with
       # the data
-      def process_results_with_block(action, resp, block)
+      def process_results_with_block(action, resp, block, aggregate)
         @stats.node_responded(resp[:senderid])
 
+        result = rpc_result_from_reply(@agent, action, resp)
+        aggregate = aggregate_reply(result, aggregate) if aggregate
+
         if resp[:body][:statuscode] == 0 || resp[:body][:statuscode] == 1
+          @stats.ok if resp[:body][:statuscode] == 0
+          @stats.fail if resp[:body][:statuscode] == 1
           @stats.time_block_execution :start
 
           case block.arity
-          when 1
-            block.call(resp)
-          when 2
-            rpcresp = Result.new(@agent, action, {:sender => resp[:senderid], :statuscode => resp[:body][:statuscode],
-                                   :statusmsg => resp[:body][:statusmsg], :data => resp[:body][:data]})
-            block.call(resp, rpcresp)
+            when 1
+              block.call(resp)
+            when 2
+              block.call(resp, result)
           end
 
           @stats.time_block_execution :end
         else
+          @stats.fail
+
           case resp[:body][:statuscode]
-          when 2
-            raise UnknownRPCAction, resp[:body][:statusmsg]
-          when 3
-            raise MissingRPCData, resp[:body][:statusmsg]
-          when 4
-            raise InvalidRPCData, resp[:body][:statusmsg]
-          when 5
-            raise UnknownRPCError, resp[:body][:statusmsg]
+            when 2
+              raise UnknownRPCAction, resp[:body][:statusmsg]
+            when 3
+              raise MissingRPCData, resp[:body][:statusmsg]
+            when 4
+              raise InvalidRPCData, resp[:body][:statusmsg]
+            when 5
+              raise UnknownRPCError, resp[:body][:statusmsg]
           end
         end
+
+        return aggregate
       end
     end
   end