mcollective-client 2.5.3 → 2.6.0

data/lib/mcollective.rb

@@ -59,7 +59,7 @@ module MCollective
 
   MCollective::Vendor.load_vendored
 
-  VERSION="2.5.3"
+  VERSION="@DEVELOPMENT_VERSION@"
 
   def self.version
     VERSION

data/lib/mcollective/application.rb

@@ -314,12 +314,27 @@ module MCollective
                        :okcount => 0,
                        :failcount => 0}.merge(stats.to_hash)
 
-      return 4 if request_stats[:discoverytime] == 0 && request_stats[:responses] == 0
-      return 3 if request_stats[:discovered] > 0 && request_stats[:responses] == 0
-      return 2 if request_stats[:discovered] > 0 && request_stats[:failcount] > 0
-      return 1 if request_stats[:discovered] == 0
-      return 0 if request_stats[:discoverytime] == 0 && request_stats[:discovered] == request_stats[:okcount]
-      return 0 if request_stats[:discovered] == request_stats[:okcount]
+      if (request_stats[:discoverytime] == 0 && request_stats[:responses] == 0)
+        return 4
+      end
+
+      if (request_stats[:discovered] > 0)
+        if (request_stats[:responses] == 0)
+          return 3
+        elsif (request_stats[:failcount] > 0)
+          return 2
+        end
+      end
+
+      if (request_stats[:discovered] == 0)
+        if (request_stats[:responses] && request_stats[:responses] > 0)
+          return 0
+        else
+          return 1
+        end
+      end
+
+      return 0
     end
 
     # A helper that creates a consistent exit code for applications by looking at an

data/lib/mcollective/client.rb

@@ -18,6 +18,11 @@ module MCollective
       @connection.connect
     end
 
+    @@request_sequence = 0
+    def self.request_sequence
+      @@request_sequence
+    end
+
     # Returns the configured main collective if no
     # specific collective is specified as options
     def collective
@@ -55,6 +60,8 @@ module MCollective
         request.reply_to = @options[:reply_to] if @options[:reply_to]
       end
 
+      @@request_sequence += 1
+
       request.encode!
       subscribe(agent, :reply) unless request.reply_to
       request

data/lib/mcollective/config.rb

@@ -15,7 +15,8 @@ module MCollective
     attr_reader :main_collective, :ssl_cipher, :registration_collective
     attr_reader :direct_addressing, :direct_addressing_threshold, :ttl
     attr_reader :default_discovery_method, :default_discovery_options
-    attr_reader :publish_timeout, :threaded, :soft_shutdown
+    attr_reader :publish_timeout, :threaded, :soft_shutdown, :activate_agents
+    attr_reader :registration_splay, :discovery_timeout, :soft_shutdown_timeout
 
     def initialize
       @configured = false
@@ -43,6 +44,8 @@ module MCollective
                   @registration_collective = val
                 when "registerinterval"
                   @registerinterval = Integer(val)
+                when "registration_splay"
+                  @registration_splay = Util.str_to_bool(val)
                 when "collectives"
                   @collectives = val.split(",").map {|c| c.strip}
                 when "main_collective"
@@ -87,6 +90,8 @@ module MCollective
                   @classesfile = val
                 when /^plugin.(.+)$/
                   @pluginconf[$1] = val
+                when "discovery_timeout"
+                  @discovery_timeout = Integer(val)
                 when "publish_timeout"
                   @publish_timeout = Integer(val)
                 when "rpcaudit"
@@ -115,6 +120,10 @@ module MCollective
                   @default_discovery_method = val
                 when "soft_shutdown"
                   @soft_shutdown = Util.str_to_bool(val)
+                when "soft_shutdown_timeout"
+                  @soft_shutdown_timeout = Integer(val)
+                when "activate_agents"
+                  @activate_agents = Util.str_to_bool(val)
                 when "topicprefix", "topicsep", "queueprefix", "rpchelptemplate", "helptemplatedir"
                   Log.warn("Use of deprecated '#{key}' option.  This option is ignored and should be removed from '#{configfile}'")
                 else
@@ -165,6 +174,7 @@ module MCollective
       @registration = "Agentlist"
       @registerinterval = 0
       @registration_collective = nil
+      @registration_splay = false
       @classesfile = "/var/lib/puppet/state/classes.txt"
       @rpcaudit = false
       @rpcauditprovider = ""
@@ -193,6 +203,8 @@ module MCollective
       @publish_timeout = 2
       @threaded = false
       @soft_shutdown = false
+      @soft_shutdown_timeout = nil
+      @activate_agents = true
     end
 
     def read_plugin_config_dir(dir)

data/lib/mcollective/connector/base.rb

@@ -17,6 +17,8 @@ module MCollective
   module Connector
     class Base
       def self.inherited(klass)
+        plugin_name = klass.to_s.split("::").last.downcase
+        ddl = DDL.new(plugin_name, :connector)
         PluginManager << {:type => "connector_plugin", :class => klass.to_s}
       end
     end

data/lib/mcollective/facts/base.rb

@@ -34,11 +34,7 @@ module MCollective
               # Force reset to last known good state on empty facts
               raise "Got empty facts" if tfacts.empty?
 
-              @facts.clear
-
-              tfacts.each_pair do |key,value|
-                @facts[key.to_s] = value.to_s
-              end
+              @facts = normalize_facts(tfacts)
 
               @last_good_facts = @facts.clone
               @last_facts_load = Time.now.to_i
@@ -81,6 +77,23 @@ module MCollective
       def force_reload?
         false
       end
+
+      private
+
+      def normalize_facts(value)
+        case value
+        when Array
+          return value.map { |v| normalize_facts(v) }
+        when Hash
+          new_hash = {}
+          value.each do |k,v|
+            new_hash[k.to_s] = normalize_facts(v)
+          end
+          return new_hash
+        else
+          return value.to_s
+        end
+      end
     end
   end
 end

data/lib/mcollective/log.rb

@@ -44,6 +44,13 @@ module MCollective
         @logger.cycle_level if @configured
       end
 
+      # reopen log files
+      def reopen
+        if @configured
+          @logger.reopen
+        end
+      end
+
       # logs a message at a certain level
       def log(level, msg)
         configure unless @configured

data/lib/mcollective/logger/base.rb

@@ -34,6 +34,18 @@ module MCollective
         @active_level = level.to_sym
       end
 
+      def start
+        raise "The logging class did not supply a start method"
+      end
+
+      def log(level, from, msg)
+        raise "The logging class did not supply a log method"
+      end
+
+      def reopen
+        # reopen may not make sense to all Loggers, but we expect it of the API
+      end
+
       private
       def map_level(level)
         raise "Logger class do not know how to handle #{level} messages" unless valid_levels.include?(level.to_sym)
@@ -60,14 +72,6 @@ module MCollective
       def valid_levels
         raise "The logging class did not supply a valid_levels method"
       end
-
-      def log(level, from, msg)
-        raise "The logging class did not supply a log method"
-      end
-
-      def start
-        raise "The logging class did not supply a start method"
-      end
     end
   end
 end

data/lib/mcollective/logger/file_logger.rb

@@ -41,6 +41,13 @@ module MCollective
         # STDERR it as last resort
         STDERR.puts("#{level}: #{msg}")
       end
+
+      def reopen
+        level = @logger.level
+        @logger.close
+        start
+        @logger.level = level
+      end
     end
   end
 end

data/lib/mcollective/message.rb

@@ -200,7 +200,7 @@ module MCollective
         if msg_age > ttl
           PluginManager["global_stats"].ttlexpired
 
-          raise(MsgTTLExpired, "message #{requestid} from #{cid} created at #{msgtime} is #{msg_age} seconds old, TTL is #{ttl}")
+          raise(MsgTTLExpired, "message #{requestid} from #{cid} created at #{msgtime} is #{msg_age} seconds old, TTL is #{ttl}.  Rejecting message.")
         end
       end
 

data/lib/mcollective/optionparser.rb

@@ -177,6 +177,10 @@ module MCollective
       @parser.on("--threaded", "Start publishing requests and receiving responses in threaded mode.") do |v|
         @options[:threaded] = true
       end
+
+      @parser.on("--sort", "Sort the output of an RPC call before processing.") do |v|
+        @options[:sort] = true
+      end
     end
 
     private

data/lib/mcollective/registration/base.rb

@@ -20,16 +20,7 @@ module MCollective
         return false if interval == 0
 
         Thread.new do
-          loop do
-            begin
-              publish(body)
-
-              sleep interval
-            rescue Exception => e
-              Log.error("Sending registration message failed: #{e}")
-              sleep interval
-            end
-          end
+          publish_thread(connection)
         end
       end
 
@@ -72,6 +63,29 @@ module MCollective
           req.publish
         end
       end
+
+      def body
+        raise "Registration Plugins must implement the #body method"
+      end
+
+      private
+      def publish_thread(connnection)
+        if config.registration_splay
+            splay_delay = rand(interval)
+            Log.debug("registration_splay enabled. Registration will start in #{splay_delay} seconds")
+            sleep splay_delay
+          end
+
+          loop do
+            begin
+              publish(body)
+              sleep interval
+            rescue Exception => e
+              Log.error("Sending registration message failed: #{e}")
+              sleep interval
+            end
+          end
+      end
     end
   end
 end

data/lib/mcollective/rpc/agent.rb

@@ -138,10 +138,16 @@ module MCollective
       # end
       def self.activate?
         agent_name = self.to_s.split("::").last.downcase
+        config = Config.instance
 
         Log.debug("Starting default activation checks for #{agent_name}")
 
-        should_activate = Util.str_to_bool(Config.instance.pluginconf.fetch("#{agent_name}.activate_agent", true))
+        # Check global state to determine if agent should be loaded
+        should_activate = config.activate_agents
+
+        # Check agent specific state to determine if agent should be loaded
+        should_activate = Util.str_to_bool(config.pluginconf.fetch("#{agent_name}.activate_agent", 
+                                           should_activate))
 
         unless should_activate
           Log.debug("Found plugin configuration '#{agent_name}.activate_agent' with value '#{should_activate}'")

data/lib/mcollective/rpc/client.rb

@@ -25,7 +25,12 @@ module MCollective
           initial_options = Marshal.load(@@initial_options)
 
         else
-          oparser = MCollective::Optionparser.new({:verbose => false, :progress_bar => true, :mcollective_limit_targets => false, :batch_size => nil, :batch_sleep_time => 1}, "filter")
+          oparser = MCollective::Optionparser.new({ :verbose => false, 
+                                                    :progress_bar => true, 
+                                                    :mcollective_limit_targets => false, 
+                                                    :batch_size => nil, 
+                                                    :batch_sleep_time => 1 }, 
+                                                  "filter")
 
           initial_options = oparser.parse do |parser, opts|
             if block_given?
@@ -67,13 +72,13 @@ module MCollective
         @discovery_options = initial_options[:discovery_options] || []
         @force_display_mode = initial_options[:force_display_mode] || false
 
-        @batch_size = Integer(initial_options[:batch_size] || 0)
+        @batch_size = initial_options[:batch_size] || 0
         @batch_sleep_time = Float(initial_options[:batch_sleep_time] || 1)
-        @batch_mode = @batch_size > 0
+        @batch_mode = determine_batch_mode(@batch_size)
 
         agent_filter agent
 
-        @discovery_timeout = @initial_options.fetch(:disctimeout, nil)
+        @discovery_timeout = @initial_options.fetch(:disctimeout, nil) || Config.instance.discovery_timeout
 
         @collective = @client.collective
         @ttl = initial_options[:ttl] || Config.instance.ttl
@@ -239,6 +244,11 @@ module MCollective
 
         validate_request(action, args)
 
+        # TODO(ploubser): The logic here seems poor. It implies that it is valid to
+        # pass arguments where batch_mode is set to false and batch_mode > 0.
+        # If this is the case we completely ignore the supplied value of batch_mode
+        # and do our own thing. 
+
         # if a global batch size is set just use that else set it
         # in the case that it was passed as an argument
         batch_mode = args.include?(:batch_size) || @batch_mode
@@ -248,7 +258,7 @@ module MCollective
         # if we were given a batch_size argument thats 0 and batch_mode was
         # determined to be on via global options etc this will allow a batch_size
         # of 0 to disable or batch_mode for this call only
-        batch_mode = (batch_mode && Integer(batch_size) > 0)
+        batch_mode = determine_batch_mode(batch_size)
 
         # Handle single target requests by doing discovery and picking
         # a random node.  Then do a custom request specifying a filter
@@ -580,7 +590,13 @@ module MCollective
 
       # Sets and sanity checks the limit_targets variable
       # used to restrict how many nodes we'll target
+      # Limit targets can be reset by passing nil or false
       def limit_targets=(limit)
+        if !limit
+          @limit_targets = nil
+          return
+        end
+
         if limit.is_a?(String)
           raise "Invalid limit specified: #{limit} valid limits are /^\d+%*$/" unless limit =~ /^\d+%*$/
 
@@ -606,10 +622,14 @@ module MCollective
 
       # Sets the batch size, if the size is set to 0 that will disable batch mode
       def batch_size=(limit)
-        raise "Can only set batch size if direct addressing is supported" unless Config.instance.direct_addressing
+        unless Config.instance.direct_addressing
+          raise "Can only set batch size if direct addressing is supported"
+        end
+        
+        validate_batch_size(limit)
 
-        @batch_size = Integer(limit)
-        @batch_mode = @batch_size > 0
+        @batch_size = limit
+        @batch_mode = determine_batch_mode(@batch_size)
       end
 
       def batch_sleep_time=(time)
@@ -759,8 +779,8 @@ module MCollective
       def call_agent_batched(action, args, opts, batch_size, sleep_time, &block)
         raise "Batched requests requires direct addressing" unless Config.instance.direct_addressing
         raise "Cannot bypass result processing for batched requests" if args[:process_results] == false
+        validate_batch_size(batch_size)
 
-        batch_size = Integer(batch_size)
         sleep_time = Float(sleep_time)
 
         Log.debug("Calling #{agent}##{action} in batches of #{batch_size} with sleep time of #{sleep_time}")
@@ -782,10 +802,22 @@ module MCollective
             @stdout.print twirl.twirl(respcount, discovered.size)
           end
 
+          if (batch_size =~ /^(\d+)%$/)
+            # determine batch_size as a percentage of the discovered array's size
+            batch_size = (discovered.size / 100.0 * Integer($1)).ceil
+          else
+            batch_size = Integer(batch_size)
+          end
+
           @stats.requestid = nil
+          processed_nodes = 0
 
-          discovered.in_groups_of(batch_size) do |hosts, last_batch|
-            message = Message.new(req, nil, {:agent => @agent, :type => :direct_request, :collective => @collective, :filter => opts[:filter], :options => opts})
+          discovered.in_groups_of(batch_size) do |hosts|
+            message = Message.new(req, nil, {:agent => @agent, 
+                                             :type => :direct_request, 
+                                             :collective => @collective, 
+                                             :filter => opts[:filter], 
+                                             :options => opts})
 
             # first time round we let the Message object create a request id
             # we then re-use it for future requests to keep auditing sane etc
@@ -808,13 +840,20 @@ module MCollective
               end
             end
 
+            if @initial_options[:sort]
+              results.sort!
+            end
+
             @stats.noresponsefrom.concat @client.stats[:noresponsefrom]
             @stats.responses += @client.stats[:responses]
             @stats.blocktime += @client.stats[:blocktime] + sleep_time
             @stats.totaltime += @client.stats[:totaltime]
             @stats.discoverytime += @client.stats[:discoverytime]
 
-            sleep sleep_time unless last_batch
+            processed_nodes += hosts.length
+            if (discovered.length > processed_nodes)
+              sleep sleep_time
+            end
           end
 
           @stats.aggregate_summary = aggregate.summarize if aggregate
@@ -898,6 +937,10 @@ module MCollective
             end
           end
 
+          if @initial_options[:sort]
+            results.sort!
+          end
+
           @stats.aggregate_summary = aggregate.summarize if aggregate
           @stats.aggregate_failures = aggregate.failed if aggregate
           @stats.client_stats = @client.stats
@@ -947,35 +990,46 @@ module MCollective
         result = rpc_result_from_reply(@agent, action, resp)
         aggregate = aggregate_reply(result, aggregate) if aggregate
 
-        if resp[:body][:statuscode] == 0 || resp[:body][:statuscode] == 1
-          @stats.ok if resp[:body][:statuscode] == 0
-          @stats.fail if resp[:body][:statuscode] == 1
-          @stats.time_block_execution :start
+        @stats.ok if resp[:body][:statuscode] == 0
+        @stats.fail if resp[:body][:statuscode] != 0
+        @stats.time_block_execution :start
 
-          case block.arity
-            when 1
-              block.call(resp)
-            when 2
-              block.call(resp, result)
-          end
+        case block.arity
+          when 1
+            block.call(resp)
+          when 2
+            block.call(resp, result)
+        end
 
-          @stats.time_block_execution :end
-        else
-          @stats.fail
+        @stats.time_block_execution :end
 
-          case resp[:body][:statuscode]
-            when 2
-              raise UnknownRPCAction, resp[:body][:statusmsg]
-            when 3
-              raise MissingRPCData, resp[:body][:statusmsg]
-            when 4
-              raise InvalidRPCData, resp[:body][:statusmsg]
-            when 5
-              raise UnknownRPCError, resp[:body][:statusmsg]
+        return aggregate
+      end
+
+      private
+      
+      def determine_batch_mode(batch_size)
+        if (batch_size != 0 && batch_size != "0")
+          return true
+        end
+
+        return false
+      end
+
+      # Validate the bach_size based on the following criteria
+      # batch_size is percentage string and it's more than 0 percent
+      # batch_size is a string of digits
+      # batch_size is of type Integer
+      def validate_batch_size(batch_size)
+        if (batch_size.is_a?(Integer))
+          return
+        elsif (batch_size.is_a?(String))
+          if ((batch_size =~ /^(\d+)%$/ && Integer($1) != 0) || batch_size =~ /^(\d+)$/)
+            return
           end
         end
 
-        return aggregate
+        raise("batch_size must be an integer or match a percentage string (e.g. '24%'")
       end
     end
   end