mcollective-client 2.2.4 → 2.4.0

data/lib/mcollective/rpc/agent.rb

@@ -65,7 +65,7 @@ module MCollective
 
         begin
           # Incoming requests need to be validated against the DDL thus reusing
-          # all the work users put into creating DDLs and creating a consistant
+          # all the work users put into creating DDLs and creating a consistent
           # quality of input validation everywhere with the a simple once off
           # investment of writing a DDL
           @request.validate!
@@ -85,7 +85,7 @@ module MCollective
           if respond_to?("#{@request.action}_action")
             send("#{@request.action}_action")
           else
-            raise UnknownRPCAction, "Unknown action: #{@request.action}"
+            raise UnknownRPCAction, "Unknown action '#{@request.action}' for agent '#{@request.agent}'"
           end
         rescue RPCAborted => e
           @reply.fail e.to_s, 1
@@ -96,13 +96,17 @@ module MCollective
         rescue MissingRPCData => e
           @reply.fail e.to_s, 3
 
-        rescue InvalidRPCData => e
+        rescue InvalidRPCData, DDLValidationError => e
           @reply.fail e.to_s, 4
 
         rescue UnknownRPCError => e
+          Log.error("%s#%s failed: %s: %s" % [@agent_name, @request.action, e.class, e.to_s])
+          Log.error(e.backtrace.join("\n\t"))
           @reply.fail e.to_s, 5
 
         rescue Exception => e
+          Log.error("%s#%s failed: %s: %s" % [@agent_name, @request.action, e.class, e.to_s])
+          Log.error(e.backtrace.join("\n\t"))
           @reply.fail e.to_s, 5
 
         end
@@ -137,16 +141,13 @@ module MCollective
 
         Log.debug("Starting default activation checks for #{agent_name}")
 
-        should_activate = Config.instance.pluginconf["#{agent_name}.activate_agent"]
+        should_activate = Util.str_to_bool(Config.instance.pluginconf.fetch("#{agent_name}.activate_agent", true))
 
-        if should_activate
-          Log.debug("Found plugin config #{agent_name}.activate_agent with value #{should_activate}")
-          unless should_activate =~ /^1|y|true$/
-            return false
-          end
+        unless should_activate
+          Log.debug("Found plugin configuration '#{agent_name}.activate_agent' with value '#{should_activate}'")
         end
 
-        return true
+        return should_activate
       end
 
       # Returns an array of actions this agent support
@@ -211,7 +212,7 @@ module MCollective
           end
         end
 
-        [:stdin, :cwd, :environment].each do |k|
+        [:stdin, :cwd, :environment, :timeout].each do |k|
           if options.include?(k)
             shellopts[k] = options[k]
           end
@@ -231,7 +232,9 @@ module MCollective
 
       # Registers meta data for the introspection hash
       def self.metadata(data)
-        Log.warn("%s: setting meta data in agents have been deprecated, DDL files are now being used for this information." % File.basename(caller.first))
+        agent = File.basename(caller.first).split(":").first
+
+        Log.warn("Setting metadata in agents has been deprecated, DDL files are now being used for this information.  Please update the '#{agent}' agent")
       end
 
       # Creates the needed activate? class in a manner similar to the other

data/lib/mcollective/rpc/client.rb

@@ -5,7 +5,7 @@ module MCollective
     class Client
       attr_accessor :timeout, :verbose, :filter, :config, :progress, :ttl, :reply_to
       attr_reader :client, :stats, :ddl, :agent, :limit_targets, :limit_method, :output_format, :batch_size, :batch_sleep_time, :batch_mode
-      attr_reader :discovery_options, :discovery_method, :limit_seed
+      attr_reader :discovery_options, :discovery_method, :default_discovery_method, :limit_seed
 
       @@initial_options = nil
 
@@ -57,7 +57,13 @@ module MCollective
         @output_format = initial_options[:output_format] || :console
         @force_direct_request = false
         @reply_to = initial_options[:reply_to]
-        @discovery_method = initial_options[:discovery_method] || Config.instance.default_discovery_method
+        @discovery_method = initial_options[:discovery_method]
+        if !@discovery_method
+          @discovery_method = Config.instance.default_discovery_method
+          @default_discovery_method = true
+        else
+          @default_discovery_method = false
+        end
         @discovery_options = initial_options[:discovery_options] || []
         @force_display_mode = initial_options[:force_display_mode] || false
 
@@ -71,6 +77,8 @@ module MCollective
 
         @collective = @client.collective
         @ttl = initial_options[:ttl] || Config.instance.ttl
+        @publish_timeout = initial_options[:publish_timeout] || Config.instance.publish_timeout
+        @threaded = initial_options[:threaded] || Config.instance.threaded
 
         # if we can find a DDL for the service override
         # the timeout of the client so we always magically
@@ -149,6 +157,18 @@ module MCollective
          :data   => data}
       end
 
+      # For the provided arguments and action the input arguments get
+      # modified by supplying any defaults provided in the DDL for arguments
+      # that were not supplied in the request
+      #
+      # We then pass the modified arguments to the DDL for validation
+      def validate_request(action, args)
+        raise "No DDL found for agent %s cannot validate inputs" % @agent unless @ddl
+
+        @ddl.set_default_input_arguments(action, args)
+        @ddl.validate_rpc_request(action, args)
+      end
+
       # Magic handler to invoke remote methods
       #
       # Once the stub is created using the constructor or the RPC#rpcclient helper you can
@@ -217,7 +237,7 @@ module MCollective
 
         @stats.reset
 
-        @ddl.validate_rpc_request(action, args) if @ddl
+        validate_request(action, args)
 
         # if a global batch size is set just use that else set it
         # in the case that it was passed as an argument
@@ -274,7 +294,7 @@ module MCollective
       # request which technically does not need filters.  If you try to use this
       # mode with direct addressing disabled an exception will be raise
       def custom_request(action, args, expected_agents, filter = {}, &block)
-        @ddl.validate_rpc_request(action, args) if @ddl
+        validate_request(action, args)
 
         if filter == {} && !Config.instance.direct_addressing
           raise "Attempted to do a filterless custom_request without direct_addressing enabled, preventing unexpected call to all nodes"
@@ -355,6 +375,7 @@ module MCollective
       # since that is the user supplied timeout either via initial options
       # or via specifically setting it on the client.
       def discovery_method=(method)
+        @default_discovery_method = false
         @discovery_method = method
 
         if @initial_options[:discovery_options]
@@ -375,7 +396,7 @@ module MCollective
 
       # Sets the class filter
       def class_filter(klass)
-        @filter["cf_class"] << klass
+        @filter["cf_class"] = @filter["cf_class"] | [klass]
         @filter["cf_class"].compact!
         reset
       end
@@ -387,10 +408,10 @@ module MCollective
 
         if value.nil?
           parsed = Util.parse_fact_string(fact)
-          @filter["fact"] << parsed unless parsed == false
+          @filter["fact"] = @filter["fact"] | [parsed] unless parsed == false
         else
           parsed = Util.parse_fact_string("#{fact}#{operator}#{value}")
-          @filter["fact"] << parsed unless parsed == false
+          @filter["fact"] = @filter["fact"] | [parsed] unless parsed == false
         end
 
         @filter["fact"].compact!
@@ -399,21 +420,21 @@ module MCollective
 
       # Sets the agent filter
       def agent_filter(agent)
-        @filter["agent"] << agent
+        @filter["agent"] = @filter["agent"] | [agent]
         @filter["agent"].compact!
         reset
       end
 
       # Sets the identity filter
       def identity_filter(identity)
-        @filter["identity"] << identity
+        @filter["identity"] = @filter["identity"] | [identity]
         @filter["identity"].compact!
         reset
       end
 
       # Set a compound filter
       def compound_filter(filter)
-        @filter["compound"] <<  Matcher.create_compound_callstack(filter)
+        @filter["compound"] = @filter["compound"] |  [Matcher.create_compound_callstack(filter)]
         reset
       end
 
@@ -479,24 +500,8 @@ module MCollective
             @discovered_agents = hosts
             @force_direct_request = true
 
-          # if an identity filter is supplied and it is all strings no regex we can use that
-          # as discovery data, technically the identity filter is then redundant if we are
-          # in direct addressing mode and we could empty it out but this use case should
-          # only really be for a few -I's on the CLI
-          #
-          # For safety we leave the filter in place for now, that way we can support this
-          # enhancement also in broadcast mode.
-          #
-          # This is only needed for the 'mc' discovery method, other methods might change
-          # the concept of identity to mean something else so we should pass the full
-          # identity filter to them
-          elsif options[:filter]["identity"].size > 0 && @discovery_method == "mc"
-            regex_filters = options[:filter]["identity"].select{|i| i.match("^\/")}.size
-
-            if regex_filters == 0
-              @discovered_agents = options[:filter]["identity"].clone
-              @force_direct_request = true if Config.instance.direct_addressing
-            end
+          else
+            identity_filter_discovery_optimization
           end
         end
 
@@ -559,7 +564,9 @@ module MCollective
          :discovery_method => @discovery_method,
          :discovery_options => @discovery_options,
          :force_display_mode => @force_display_mode,
-         :config => @config}
+         :config => @config,
+         :publish_timeout => @publish_timeout,
+         :threaded => @threaded}
       end
 
       # Sets the collective we are communicating with
@@ -701,7 +708,9 @@ module MCollective
       #
       # Should only be called via method_missing
       def fire_and_forget_request(action, args, filter=nil)
-        @ddl.validate_rpc_request(action, args) if @ddl
+        validate_request(action, args)
+
+        identity_filter_discovery_optimization
 
         req = new_request(action.to_s, args)
 
@@ -710,7 +719,34 @@ module MCollective
         message = Message.new(req, nil, {:agent => @agent, :type => :request, :collective => @collective, :filter => filter, :options => options})
         message.reply_to = @reply_to if @reply_to
 
-        return @client.sendreq(message, nil)
+        if @force_direct_request || @client.discoverer.force_direct_mode?
+          message.discovered_hosts = discover.clone
+          message.type = :direct_request
+        end
+
+        client.sendreq(message, nil)
+      end
+
+      # if an identity filter is supplied and it is all strings no regex we can use that
+      # as discovery data, technically the identity filter is then redundant if we are
+      # in direct addressing mode and we could empty it out but this use case should
+      # only really be for a few -I's on the CLI
+      #
+      # For safety we leave the filter in place for now, that way we can support this
+      # enhancement also in broadcast mode.
+      #
+      # This is only needed for the 'mc' discovery method, other methods might change
+      # the concept of identity to mean something else so we should pass the full
+      # identity filter to them
+      def identity_filter_discovery_optimization
+        if options[:filter]["identity"].size > 0 && @discovery_method == "mc"
+          regex_filters = options[:filter]["identity"].select{|i| i.match("^\/")}.size
+
+          if regex_filters == 0
+            @discovered_agents = options[:filter]["identity"].clone
+            @force_direct_request = true if Config.instance.direct_addressing
+          end
+        end
       end
 
       # Calls an agent in a way very similar to call_agent but it supports batching

data/lib/mcollective/rpc/helpers.rb

@@ -100,6 +100,7 @@ module MCollective
                     result_text << text_for_result(sender, status, message, result, ddl)
 
                   when :flatten
+                    Log.warn("The display option :flatten is being deprecated and will be removed in the next minor release")
                     result_text << text_for_flattened_result(status, result)
 
                 end
@@ -128,7 +129,7 @@ module MCollective
         result_text << "   %s\n" % [Util.colorize(:yellow, msg)] unless msg == "OK"
 
         # only print good data, ignore data that results from failure
-        if [0, 1].include?(status)
+        if status == 0
           if result.is_a?(Hash)
             # figure out the lengths of the display as strings, we'll use
             # it later to correctly justify the output
@@ -172,6 +173,10 @@ module MCollective
                 result_text << " " << result[k].pretty_inspect.split("\n").join("\n" << padding) << "\n"
               end
             end
+          elsif status == 1
+            # for status 1 we dont want to show half baked
+            # data by default since the DDL will supply all the defaults
+            # it just doesnt look right
           else
             result_text << "\n\t" + result.pretty_inspect.split("\n").join("\n\t")
           end
@@ -249,6 +254,7 @@ module MCollective
       # Add SimpleRPC common options
       def self.add_simplerpc_options(parser, options)
         parser.separator ""
+        parser.separator "RPC Options"
 
         # add SimpleRPC specific options to all clients that use our library
         parser.on('--np', '--no-progress', 'Do not show the progress bar') do |v|

data/lib/mcollective/rpc/reply.rb

@@ -26,7 +26,9 @@ module MCollective
         interface = @ddl.action_interface(@action)
 
         interface[:output].keys.each do |output|
-          @data[output] = interface[:output][output][:default]
+          # must deep clone this data to avoid accidental updates of the DDL in cases where the
+          # default is for example a string and someone does << on it
+          @data[output] = Marshal.load(Marshal.dump(interface[:output][output].fetch(:default, nil)))
         end
       end
 

data/lib/mcollective/shell.rb

@@ -17,9 +17,12 @@ module MCollective
   #   stderr      - a variable that will receive stdin, must support <<
   #   environment - the shell environment, defaults to include LC_ALL=C
   #                 set to nil to clear the environment even of LC_ALL
+  #   timeout     - a timeout in seconds after which the subprocess is killed,
+  #                 the special value :on_thread_exit kills the subprocess
+  #                 when the invoking thread (typically the agent) has ended
   #
   class Shell
-    attr_reader :environment, :command, :status, :stdout, :stderr, :stdin, :cwd
+    attr_reader :environment, :command, :status, :stdout, :stderr, :stdin, :cwd, :timeout
 
     def initialize(command, options={})
       @environment = {"LC_ALL" => "C"}
@@ -29,6 +32,7 @@ module MCollective
       @stderr = ""
       @stdin = nil
       @cwd = Dir.tmpdir
+      @timeout = nil
 
       options.each do |opt, val|
         case opt.to_s
@@ -54,6 +58,9 @@ module MCollective
             else
               @environment.merge!(val.dup)
             end
+          when "timeout"
+            raise "timeout should be a positive integer or the symbol :on_thread_exit symbol" unless val.eql?(:on_thread_exit) || ( val.is_a?(Fixnum) && val>0 )
+            @timeout = val
         end
       end
     end
@@ -67,21 +74,51 @@ module MCollective
 
       opts["stdin"] = @stdin if @stdin
 
-      # Running waitpid on the cid here will start a thread
-      # with the waitpid in it, this way even if the thread
-      # that started this process gets killed due to agent
-      # timeout or such there will still be a waitpid waiting
-      # for the child to exit and not leave zombies.
+
+      thread = Thread.current
+      # Start a double fork and exec with systemu which implies a guard thread.
+      # If a valid timeout is configured the guard thread will terminate the
+      # executing process and reap the pid.
+      # If no timeout is specified the process will run to completion with the
+      # guard thread reaping the pid on completion.
       @status = systemu(@command, opts) do |cid|
         begin
-          sleep 1
-          Process::waitpid(cid)
+          if timeout.is_a?(Fixnum)
+            # wait for the specified timeout
+            sleep timeout
+          else
+            # sleep while the agent thread is still alive
+            while(thread.alive?)
+              sleep 0.1
+            end
+          end
+
+          # if the process is still running
+          if (Process.kill(0, cid))
+            # and a timeout was specified
+            if timeout
+              if Util.windows?
+                Process.kill('KILL', cid)
+              else
+                # Kill the process
+                Process.kill('TERM', cid)
+                sleep 2
+                Process.kill('KILL', cid) if (Process.kill(0, cid))
+              end
+            end
+            # only wait if the parent thread is dead
+            Process.waitpid(cid) unless thread.alive?
+          end
         rescue SystemExit
+        rescue Errno::ESRCH
         rescue Errno::ECHILD
+          Log.warn("Could not reap process '#{cid}'.")
         rescue Exception => e
           Log.info("Unexpected exception received while waiting for child process: #{e.class}: #{e}")
         end
       end
+      @status.thread.kill
+      @status
     end
   end
 end

data/lib/mcollective/util.rb

@@ -430,7 +430,7 @@ module MCollective
     # Code originally from Puppet
     def self.versioncmp(version_a, version_b)
       vre = /[-.]|\d+|[^-.\d]+/
-        ax = version_a.scan(vre)
+      ax = version_a.scan(vre)
       bx = version_b.scan(vre)
 
       while (ax.length>0 && bx.length>0)
@@ -456,5 +456,41 @@ module MCollective
 
       version_a <=> version_b;
     end
+
+    # we should really use Pathname#absolute? but it's not in all the
+    # ruby versions we support and it comes down to roughly this
+    def self.absolute_path?(path, separator=File::SEPARATOR, alt_separator=File::ALT_SEPARATOR)
+      if alt_separator
+        path_matcher = /^([a-zA-Z]:){0,1}[#{Regexp.quote alt_separator}#{Regexp.quote separator}]/
+      else
+        path_matcher = /^#{Regexp.quote separator}/
+      end
+
+      !!path.match(path_matcher)
+    end
+
+    # Converts a string into a boolean value
+    # Strings matching 1,y,yes,true or t will return TrueClass
+    # Any other value will return FalseClass
+    def self.str_to_bool(val)
+      clean_val = val.to_s.strip
+      if clean_val =~ /^(1|yes|true|y|t)$/i
+        return  true
+      elsif clean_val =~ /^(0|no|false|n|f)$/i
+        return false
+      else
+        raise("Cannot convert string value '#{clean_val}' into a boolean.")
+      end
+    end
+
+    # Looks up the template directory and returns its full path
+    def self.templatepath(template_file)
+      config_dir = File.dirname(Config.instance.configfile)
+      template_path = File.join(config_dir, template_file)
+      return template_path if File.exists?(template_path)
+
+      template_path = File.join("/etc/mcollective", template_file)
+      return template_path
+    end
   end
 end