mcollective-client 2.2.4 → 2.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 56381fee132b101d5841676045d4294419404fba
+  data.tar.gz: cfeb66a43d1179d79489d9dfef032d936bdda499
+SHA512:
+  metadata.gz: 44e8c61ac26956818a744e4a3daa1b5513a6da8c80b25747d9864d2394825337438817eeab07b8edd195947ea123d07310d1479df84c1ad09e75823a94e71998
+  data.tar.gz: b09053846889112bdad071459484f8511418a0fef51e1a26d3d2b39a04b34f7e2d9d87752ff0beb96785f3acd463d9b0a69234f892ca0a6177458895ca824090

data/lib/mcollective/application.rb

@@ -149,6 +149,8 @@ module MCollective
           parser.separator ""
         end
 
+        parser.separator "Application Options" unless application_cli_arguments.empty?
+
         parser.define_tail ""
         parser.define_tail "The Marionette Collective #{MCollective.version}"
 
@@ -180,7 +182,7 @@ module MCollective
             parser.send(*opts_array) do |v|
               validate_option(carg[:validate], carg[:name], v)
 
-              configuration[carg[:name]] = true
+              configuration[carg[:name]] = v
             end
 
           elsif carg[:type] == :array
@@ -255,7 +257,11 @@ module MCollective
         raise(e)
       end
 
-      err_dest.puts "\nThe %s application failed to run, use -v for full error details: %s\n" % [ Util.colorize(:bold, $0), Util.colorize(:red, e.to_s)]
+      if options[:verbose]
+        err_dest.puts "\nThe %s application failed to run: %s\n" % [ Util.colorize(:bold, $0), Util.colorize(:red, e.to_s)]
+      else
+        err_dest.puts "\nThe %s application failed to run, use -v for full error backtrace details: %s\n" % [ Util.colorize(:bold, $0), Util.colorize(:red, e.to_s)]
+      end
 
       if options.nil? || options[:verbose]
         e.backtrace.first << Util.colorize(:red, "  <----")
@@ -302,46 +308,31 @@ module MCollective
       exit 1
     end
 
+    def halt_code(stats)
+      request_stats = {:discoverytime => 0,
+                       :discovered => 0,
+                       :okcount => 0,
+                       :failcount => 0}.merge(stats.to_hash)
+
+      return 4 if request_stats[:discoverytime] == 0 && request_stats[:responses] == 0
+      return 3 if request_stats[:discovered] > 0 && request_stats[:responses] == 0
+      return 2 if request_stats[:discovered] > 0 && request_stats[:failcount] > 0
+      return 1 if request_stats[:discovered] == 0
+      return 0 if request_stats[:discoverytime] == 0 && request_stats[:discovered] == request_stats[:okcount]
+      return 0 if request_stats[:discovered] == request_stats[:okcount]
+    end
+
     # A helper that creates a consistent exit code for applications by looking at an
     # instance of MCollective::RPC::Stats
     #
     # Exit with 0 if nodes were discovered and all passed
-    # Exit with 0 if no discovery were done and > 0 responses were received
+    # Exit with 0 if no discovery were done and > 0 responses were received, all ok
     # Exit with 1 if no nodes were discovered
     # Exit with 2 if nodes were discovered but some RPC requests failed
-    # Exit with 3 if nodes were discovered, but not responses received
+    # Exit with 3 if nodes were discovered, but no responses received
     # Exit with 4 if no discovery were done and no responses were received
     def halt(stats)
-      request_stats = {:discoverytime => 0,
-                       :discovered => 0,
-                       :failcount => 0}.merge(stats.to_hash)
-
-      # was discovery done?
-      if request_stats[:discoverytime] != 0
-        # was any nodes discovered
-        if request_stats[:discovered] == 0
-          exit 1
-
-        # nodes were discovered, did we get responses
-        elsif request_stats[:responses] == 0
-          exit 3
-
-        else
-          # we got responses and discovery was done, no failures
-          if request_stats[:failcount] == 0
-            exit 0
-          else
-            exit 2
-          end
-        end
-      else
-        # discovery wasnt done and we got no responses
-        if request_stats[:responses] == 0
-          exit 4
-        else
-          exit 0
-        end
-      end
+      exit(halt_code(stats))
     end
 
     # Wrapper around MC::RPC#rpcclient that forcably supplies our options hash

data/lib/mcollective/client.rb

@@ -37,6 +37,15 @@ module MCollective
     # Sends a request and returns the generated request id, doesn't wait for
     # responses and doesn't execute any passed in code blocks for responses
     def sendreq(msg, agent, filter = {})
+      request = createreq(msg, agent, filter)
+
+      Log.debug("Sending request #{request.requestid} to the #{request.agent} agent with ttl #{request.ttl} in collective #{request.collective}")
+
+      request.publish
+      request.requestid
+    end
+
+    def createreq(msg, agent, filter ={})
       if msg.is_a?(Message)
         request = msg
         agent = request.agent
@@ -47,14 +56,8 @@ module MCollective
       end
 
       request.encode!
-
-      Log.debug("Sending request #{request.requestid} to the #{request.agent} agent with ttl #{request.ttl} in collective #{request.collective}")
-
       subscribe(agent, :reply) unless request.reply_to
-
-      request.publish
-
-      request.requestid
+      request
     end
 
     def subscribe(agent, type)
@@ -91,15 +94,14 @@ module MCollective
         reply.expected_msgid = requestid
 
         reply.decode!
-
-        reply.payload[:senderid] = Digest::MD5.hexdigest(reply.payload[:senderid]) if ENV.include?("MCOLLECTIVE_ANON")
-
-        raise(MsgDoesNotMatchRequestID, "Message reqid #{requestid} does not match our reqid #{reply.requestid}") unless reply.requestid == requestid
+        unless reply.requestid == requestid
+          raise(MsgDoesNotMatchRequestID, "Message reqid #{reply.requestid} does not match our reqid #{requestid}")
+        end
       rescue SecurityValidationFailed => e
         Log.warn("Ignoring a message that did not pass security validations")
         retry
       rescue MsgDoesNotMatchRequestID => e
-        Log.debug("Ignoring a message for some other client")
+        Log.debug("Ignoring a message for some other client : #{e.message}")
         retry
       end
 
@@ -124,7 +126,7 @@ module MCollective
     #
     # It returns a hash of times and timeouts for discovery and total run is taken from the options
     # hash which in turn is generally built using MCollective::Optionparser
-    def req(body, agent=nil, options=false, waitfor=0)
+    def req(body, agent=nil, options=false, waitfor=0, &block)
       if body.is_a?(Message)
         agent = body.agent
         waitfor = body.discovered_hosts.size || 0
@@ -132,46 +134,102 @@ module MCollective
       end
 
       @options = options if options
-
-      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
-
+      threaded = @options[:threaded]
       timeout = @discoverer.discovery_timeout(@options[:timeout], @options[:filter])
-
+      request = createreq(body, agent, @options[:filter])
+      publish_timeout = @options[:publish_timeout]
+      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
       STDOUT.sync = true
-
       hosts_responded = 0
-      reqid = nil
+
 
       begin
-        Log.debug("Publishing request to agent %s with timeout %d" % [agent, timeout])
+        if threaded
+          hosts_responded = threaded_req(request, publish_timeout, timeout, waitfor, &block)
+        else
+          hosts_responded = unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
+        end
+      rescue Interrupt => e
+      ensure
+        unsubscribe(agent, :reply)
+      end
 
-        Timeout.timeout(timeout) do
-          reqid = sendreq(body, agent, @options[:filter])
+      return update_stat(stat, hosts_responded, request.requestid)
+    end
 
-          loop do
-            resp = receive(reqid)
+    # Starts the client receiver and publisher unthreaded.
+    # This is the default client behaviour.
+    def unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
+      start_publisher(request, publish_timeout)
+      start_receiver(request.requestid, waitfor, timeout, &block)
+    end
 
-            hosts_responded += 1
+    # Starts the client receiver and publisher in threads.
+    # This is activated when the 'threader_client' configuration
+    # option is set.
+    def threaded_req(request, publish_timeout, timeout, waitfor, &block)
+      Log.debug("Starting threaded client")
+      publisher = Thread.new do
+        start_publisher(request, publish_timeout)
+      end
 
-            yield(resp.payload)
+      # When the client is threaded we add the publishing timeout to
+      # the agent timeout so that the receiver doesn't time out before
+      # publishing has finished in cases where publish_timeout >= timeout.
+      total_timeout = publish_timeout + timeout
+      hosts_responded = 0
+
+      receiver = Thread.new do
+        hosts_responded = start_receiver(request.requestid, waitfor, total_timeout, &block)
+      end
 
-            break if (waitfor != 0 && hosts_responded >= waitfor)
-          end
+      receiver.join
+      hosts_responded
+    end
+
+    # Starts the request publishing routine
+    def start_publisher(request, publish_timeout)
+      Log.debug("Starting publishing with publish timeout of #{publish_timeout}")
+      begin
+        Timeout.timeout(publish_timeout) do
+          Log.debug("Sending request #{request.requestid} to the #{request.agent} agent with ttl #{request.ttl} in collective #{request.collective}")
+          request.publish
         end
-      rescue Interrupt => e
       rescue Timeout::Error => e
-      ensure
-        unsubscribe(agent, :reply)
+        Log.warn("Could not publish all messages. Publishing timed out.")
+      end
+    end
+
+    # Starts the response receiver routine
+    # Expected to return the amount of received responses.
+    def start_receiver(requestid, waitfor, timeout, &block)
+      Log.debug("Starting response receiver with timeout of #{timeout}")
+      hosts_responded = 0
+      begin
+        Timeout.timeout(timeout) do
+          begin
+            resp = receive(requestid)
+            yield resp.payload
+            hosts_responded += 1
+          end while (waitfor == 0 || hosts_responded < waitfor)
+        end
+      rescue Timeout::Error => e
+        if (waitfor > hosts_responded)
+          Log.warn("Could not receive all responses. Expected : #{waitfor}. Received : #{hosts_responded}")
+        end
       end
 
+      hosts_responded
+    end
+
+    def update_stat(stat, hosts_responded, requestid)
       stat[:totaltime] = Time.now.to_f - stat[:starttime]
       stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
       stat[:responses] = hosts_responded
       stat[:noresponsefrom] = []
-      stat[:requestid] = reqid
+      stat[:requestid] = requestid
 
       @stats = stat
-      return stat
     end
 
     def discovered_req(body, agent, options=false)

data/lib/mcollective/config.rb

@@ -5,16 +5,17 @@ module MCollective
 
     attr_accessor :mode
 
-    attr_reader :topicprefix, :daemonize, :pluginconf, :libdir, :configured
+    attr_reader :daemonize, :pluginconf, :libdir, :configured
     attr_reader :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility
     attr_reader :identity, :daemonize, :connector, :securityprovider, :factsource
-    attr_reader :registration, :registerinterval, :topicsep, :classesfile
+    attr_reader :registration, :registerinterval, :classesfile
     attr_reader :rpcauditprovider, :rpcaudit, :configdir, :rpcauthprovider
-    attr_reader :rpcauthorization, :color, :configfile, :rpchelptemplate
+    attr_reader :rpcauthorization, :color, :configfile
     attr_reader :rpclimitmethod, :logger_type, :fact_cache_time, :collectives
     attr_reader :main_collective, :ssl_cipher, :registration_collective
-    attr_reader :direct_addressing, :direct_addressing_threshold, :ttl, :helptemplatedir
-    attr_reader :queueprefix, :default_discovery_method, :default_discovery_options
+    attr_reader :direct_addressing, :direct_addressing_threshold, :ttl
+    attr_reader :default_discovery_method, :default_discovery_options
+    attr_reader :publish_timeout, :threaded
 
     def initialize
       @configured = false
@@ -24,39 +25,34 @@ module MCollective
       set_config_defaults(configfile)
 
       if File.exists?(configfile)
-        File.open(configfile, "r").each do |line|
+        File.readlines(configfile).each do |line|
 
           # strip blank spaces, tabs etc off the end of all lines
           line.gsub!(/\s*$/, "")
 
           unless line =~ /^#|^$/
             if (line =~ /(.+?)\s*=\s*(.+)/)
-              key = $1
+              key = $1.strip
               val = $2
 
-              case key
-                when "topicsep"
-                  @topicsep = val
+              begin
+                case key
                 when "registration"
                   @registration = val.capitalize
                 when "registration_collective"
                   @registration_collective = val
                 when "registerinterval"
-                  @registerinterval = val.to_i
+                  @registerinterval = Integer(val)
                 when "collectives"
                   @collectives = val.split(",").map {|c| c.strip}
                 when "main_collective"
                   @main_collective = val
-                when "topicprefix"
-                  @topicprefix = val
-                when "queueprefix"
-                  @queueprefix = val
                 when "logfile"
                   @logfile = val
                 when "keeplogs"
-                  @keeplogs = val.to_i
+                  @keeplogs = Integer(val)
                 when "max_log_size"
-                  @max_log_size = val.to_i
+                  @max_log_size = Integer(val)
                 when "loglevel"
                   @loglevel = val
                 when "logfacility"
@@ -64,6 +60,8 @@ module MCollective
                 when "libdir"
                   paths = val.split(File::PATH_SEPARATOR)
                   paths.each do |path|
+                    raise("libdir paths should be absolute paths but '%s' is relative" % path) unless Util.absolute_path?(path)
+
                     @libdir << path
                     unless $LOAD_PATH.include?(path)
                       $LOAD_PATH << path
@@ -72,13 +70,13 @@ module MCollective
                 when "identity"
                   @identity = val
                 when "direct_addressing"
-                  val =~ /^1|y/i ? @direct_addressing = true : @direct_addressing = false
+                  @direct_addressing = Util.str_to_bool(val)
                 when "direct_addressing_threshold"
-                  @direct_addressing_threshold = val.to_i
+                  @direct_addressing_threshold = Integer(val)
                 when "color"
-                  val =~ /^1|y/i ? @color = true : @color = false
+                  @color = Util.str_to_bool(val)
                 when "daemonize"
-                  val =~ /^1|y/i ? @daemonize = true : @daemonize = false
+                  @daemonize = Util.str_to_bool(val)
                 when "securityprovider"
                   @securityprovider = val.capitalize
                 when "factsource"
@@ -89,39 +87,46 @@ module MCollective
                   @classesfile = val
                 when /^plugin.(.+)$/
                   @pluginconf[$1] = val
+                when "publish_timeout"
+                  @publish_timeout = Integer(val)
                 when "rpcaudit"
-                  val =~ /^1|y/i ? @rpcaudit = true : @rpcaudit = false
+                  @rpcaudit = Util.str_to_bool(val)
                 when "rpcauditprovider"
                   @rpcauditprovider = val.capitalize
                 when "rpcauthorization"
-                  val =~ /^1|y/i ? @rpcauthorization = true : @rpcauthorization = false
+                  @rpcauthorization = Util.str_to_bool(val)
                 when "rpcauthprovider"
                   @rpcauthprovider = val.capitalize
-                when "rpchelptemplate"
-                  @rpchelptemplate = val
                 when "rpclimitmethod"
                   @rpclimitmethod = val.to_sym
                 when "logger_type"
                   @logger_type = val
                 when "fact_cache_time"
-                  @fact_cache_time = val.to_i
+                  @fact_cache_time = Integer(val)
                 when "ssl_cipher"
                   @ssl_cipher = val
+                when "threaded"
+                  @threaded = Util.str_to_bool(val)
                 when "ttl"
-                  @ttl = val.to_i
-                when "helptemplatedir"
-                  @helptemplatedir = val
+                  @ttl = Integer(val)
                 when "default_discovery_options"
                   @default_discovery_options << val
                 when "default_discovery_method"
                   @default_discovery_method = val
+                when "topicprefix", "topicsep", "queueprefix", "rpchelptemplate", "helptemplatedir"
+                  Log.warn("Use of deprecated '#{key}' option.  This option is ignored and should be removed from '#{configfile}'")
                 else
-                  raise("Unknown config parameter #{key}")
+                  raise("Unknown config parameter '#{key}'")
+                end
+              rescue ArgumentError => e
+                raise "Could not parse value for configuration option '#{key}' with value '#{val}'"
               end
             end
           end
         end
 
+        raise('The %s config file does not specify a libdir setting, cannot continue' % configfile) if @libdir.empty?
+
         read_plugin_config_dir("#{@configdir}/plugin.d")
 
         raise 'Identities can only match /\w\.\-/' unless @identity.match(/^[\w\.\-]+$/)
@@ -140,6 +145,8 @@ module MCollective
         PluginManager.loadclass("Mcollective::Registration::#{@registration}")
         PluginManager.loadclass("Mcollective::Audit::#{@rpcauditprovider}") if @rpcaudit
         PluginManager << {:type => "global_stats", :class => RunnerStats.new}
+
+        Log.info("The Marionette Collective version #{MCollective::VERSION} started by #{$0} using config file #{configfile}")
       else
         raise("Cannot find config file '#{configfile}'")
       end
@@ -149,16 +156,13 @@ module MCollective
       @stomp = Hash.new
       @subscribe = Array.new
       @pluginconf = Hash.new
-      @connector = "Stomp"
+      @connector = "activemq"
       @securityprovider = "Psk"
       @factsource = "Yaml"
       @identity = Socket.gethostname
       @registration = "Agentlist"
       @registerinterval = 0
       @registration_collective = nil
-      @topicsep = "."
-      @topicprefix = "/topic/"
-      @queueprefix = "/queue/"
       @classesfile = "/var/lib/puppet/state/classes.txt"
       @rpcaudit = false
       @rpcauditprovider = ""
@@ -178,19 +182,14 @@ module MCollective
       @collectives = ["mcollective"]
       @main_collective = @collectives.first
       @ssl_cipher = "aes-256-cbc"
-      @direct_addressing = false
+      @direct_addressing = true
       @direct_addressing_threshold = 10
       @default_discovery_method = "mc"
       @default_discovery_options = []
       @ttl = 60
       @mode = :client
-
-      # look in the config dir for the template so users can provide their own and windows
-      # with odd paths will just work more often, but fall back to old behavior if it does
-      # not exist
-      @rpchelptemplate = File.join(File.dirname(configfile), "rpc-help.erb")
-      @rpchelptemplate = "/etc/mcollective/rpc-help.erb" unless File.exists?(@rpchelptemplate)
-      @helptemplatedir = File.dirname(@rpchelptemplate)
+      @publish_timeout = 2
+      @threaded = false
     end
 
     def read_plugin_config_dir(dir)
@@ -205,7 +204,7 @@ module MCollective
           line.gsub!(/\s*$/, "")
           next if line =~ /^#|^$/
           if (line =~ /(.+?)\s*=\s*(.+)/)
-            key = $1
+            key = $1.strip
             val = $2
             @pluginconf["#{plugin}.#{key}"] = val
           end