mcollective-client 1.3.3

data/lib/mcollective/applications.rb

@@ -0,0 +1,145 @@
+module MCollective
+  class Applications
+    def self.[](appname)
+      load_application(appname)
+      PluginManager["#{appname}_application"]
+    end
+
+    def self.run(appname)
+      load_config
+
+      begin
+        load_application(appname)
+      rescue Exception => e
+        e.backtrace.first << RPC::Helpers.colorize(:red, "  <----")
+        STDERR.puts "Application '#{appname}' failed to load:"
+        STDERR.puts
+        STDERR.puts RPC::Helpers.colorize(:red, "   #{e} (#{e.class})")
+        STDERR.puts
+        STDERR.puts "       %s" % [e.backtrace.join("\n       ")]
+        exit 1
+      end
+
+      PluginManager["#{appname}_application"].run
+    end
+
+    def self.load_application(appname)
+      return if PluginManager.include?("#{appname}_application")
+
+      load_config
+
+      PluginManager.loadclass "MCollective::Application::#{appname.capitalize}"
+      PluginManager << {:type => "#{appname}_application", :class => "MCollective::Application::#{appname.capitalize}"}
+    end
+
+    # Returns an array of applications found in the lib dirs
+    def self.list
+      load_config
+
+      applist = []
+
+      Config.instance.libdir.each do |libdir|
+        applicationdir = "#{libdir}/mcollective/application"
+        next unless File.directory?(applicationdir)
+
+        Dir.entries(applicationdir).grep(/\.rb$/).each do |application|
+          applist << File.basename(application, ".rb")
+        end
+      end
+
+      applist
+    rescue SystemExit
+      exit 1
+    rescue Exception => e
+      STDERR.puts "Failed to generate application list: #{e.class}: #{e}"
+      exit 1
+    end
+
+    # Filters a string of opts out using Shellwords
+    # keeping only things related to --config and -c
+    def self.filter_extra_options(opts)
+      res = ""
+      words = Shellwords.shellwords(opts)
+      words.each_with_index do |word,idx|
+        if word == "-c"
+          return "--config=#{words[idx + 1]}"
+        elsif word == "--config"
+          return "--config=#{words[idx + 1]}"
+        elsif word =~ /\-c=/
+          return word
+        elsif word =~ /\-\-config=/
+          return word
+        end
+      end
+
+      return ""
+    end
+
+    # We need to know the config file in order to know the libdir
+    # so that we can find applications.
+    #
+    # The problem is the CLI might be stuffed with options only the
+    # app in the libdir might understand so we have a chicken and
+    # egg situation.
+    #
+    # We're parsing and filtering MCOLLECTIVE_EXTRA_OPTS removing
+    # all but config related options and parsing the options looking
+    # just for the config file.
+    #
+    # We're handling failures gracefully and finally restoring the
+    # ARG and MCOLLECTIVE_EXTRA_OPTS to the state they were before
+    # we started parsing.
+    #
+    # This is mostly a hack, when we're redoing how config works
+    # this stuff should be made less sucky
+    def self.load_config
+      return if Config.instance.configured
+
+      original_argv = ARGV.clone
+      original_extra_opts = ENV["MCOLLECTIVE_EXTRA_OPTS"].clone rescue nil
+      configfile = nil
+
+      parser = OptionParser.new
+      parser.on("--config CONFIG", "-c", "Config file") do |f|
+        configfile = f
+      end
+
+      parser.program_name = $0
+
+      parser.on("--help")
+
+      # avoid option parsers own internal version handling that sux
+      parser.on("-v", "--verbose")
+
+      if original_extra_opts
+        begin
+          # optparse will parse the whole ENV in one go and refuse
+          # to play along with the retry trick I do below so in
+          # order to handle unknown options properly I parse out
+          # only -c and --config deleting everything else and
+          # then restore the environment variable later when I
+          # am done with it
+          ENV["MCOLLECTIVE_EXTRA_OPTS"] = filter_extra_options(ENV["MCOLLECTIVE_EXTRA_OPTS"].clone)
+          parser.environment("MCOLLECTIVE_EXTRA_OPTS")
+        rescue Exception => e
+          Log.error("Failed to parse MCOLLECTIVE_EXTRA_OPTS: #{e}")
+        end
+
+        ENV["MCOLLECTIVE_EXTRA_OPTS"] = original_extra_opts.clone
+      end
+
+      begin
+        parser.parse!
+      rescue OptionParser::InvalidOption => e
+        retry
+      end
+
+      ARGV.clear
+      original_argv.each {|a| ARGV << a}
+
+      configfile = Util.config_file_for_user unless configfile
+
+      Config.instance.loadconfig(configfile)
+    end
+  end
+end

data/lib/mcollective/client.rb

@@ -0,0 +1,292 @@
+module MCollective
+  # Helpers for writing clients that can talk to agents, do discovery and so forth
+  class Client
+    attr_accessor :options, :stats
+
+    def initialize(configfile)
+      @config = Config.instance
+      @config.loadconfig(configfile) unless @config.configured
+
+      @connection = PluginManager["connector_plugin"]
+      @security = PluginManager["security_plugin"]
+
+      @security.initiated_by = :client
+      @options = nil
+      @subscriptions = {}
+
+      @connection.connect
+    end
+
+    # Returns the configured main collective if no
+    # specific collective is specified as options
+    def collective
+      if @options[:collective].nil?
+        @config.main_collective
+      else
+        @options[:collective]
+      end
+    end
+
+    # Disconnects cleanly from the middleware
+    def disconnect
+      Log.debug("Disconnecting from the middleware")
+      @connection.disconnect
+    end
+
+    # Sends a request and returns the generated request id, doesn't wait for
+    # responses and doesn't execute any passed in code blocks for responses
+    def sendreq(msg, agent, filter = {})
+      if msg.is_a?(Message)
+        request = msg
+        agent = request.agent
+      else
+        ttl = @options[:ttl] || @config.ttl
+        request = Message.new(msg, nil, {:agent => agent, :type => :request, :collective => collective, :filter => filter, :ttl => ttl})
+        request.reply_to = @options[:reply_to] if @options[:reply_to]
+      end
+
+      request.encode!
+
+      Log.debug("Sending request #{request.requestid} to the #{request.agent} agent with ttl #{request.ttl} in collective #{request.collective}")
+
+      subscribe(agent, :reply)
+
+      request.publish
+
+      request.requestid
+    end
+
+    def subscribe(agent, type)
+      unless @subscriptions.include?(agent)
+        subscription = Util.make_subscriptions(agent, type, collective)
+        Log.debug("Subscribing to #{type} target for agent #{agent}")
+
+        Util.subscribe(subscription)
+        @subscriptions[agent] = 1
+      end
+    end
+
+    def unsubscribe(agent, type)
+      if @subscriptions.include?(agent)
+        subscription = Util.make_subscriptions(agent, type, collective)
+        Log.debug("Unsubscribing #{type} target for #{agent}")
+
+        Util.unsubscribe(subscription)
+        @subscriptions.delete(agent)
+      end
+    end
+    # Blocking call that waits for ever for a message to arrive.
+    #
+    # If you give it a requestid this means you've previously send a request
+    # with that ID and now you just want replies that matches that id, in that
+    # case the current connection will just ignore all messages not directed at it
+    # and keep waiting for more till it finds a matching message.
+    def receive(requestid = nil)
+      reply = nil
+
+      begin
+        reply = @connection.receive
+        reply.type = :reply
+        reply.expected_msgid = requestid
+
+        reply.decode!
+
+        reply.payload[:senderid] = Digest::MD5.hexdigest(reply.payload[:senderid]) if ENV.include?("MCOLLECTIVE_ANON")
+
+        raise(MsgDoesNotMatchRequestID, "Message reqid #{requestid} does not match our reqid #{reply.requestid}") unless reply.requestid == requestid
+      rescue SecurityValidationFailed => e
+        Log.warn("Ignoring a message that did not pass security validations")
+        retry
+      rescue MsgDoesNotMatchRequestID => e
+        Log.debug("Ignoring a message for some other client")
+        retry
+      end
+
+      reply
+    end
+
+    # Performs a discovery of nodes matching the filter passed
+    # returns an array of nodes
+    #
+    # An integer limit can be supplied this will have the effect
+    # of the discovery being cancelled soon as it reached the
+    # requested limit of hosts
+    def discover(filter, timeout, limit=0)
+      raise "Limit has to be an integer" unless limit.is_a?(Fixnum)
+
+      begin
+        hosts = []
+        Timeout.timeout(timeout) do
+          reqid = sendreq("ping", "discovery", filter)
+          Log.debug("Waiting #{timeout} seconds for discovery replies to request #{reqid}")
+
+          loop do
+            reply = receive(reqid)
+            Log.debug("Got discovery reply from #{reply.payload[:senderid]}")
+            hosts << reply.payload[:senderid]
+
+            return hosts if limit > 0 && hosts.size == limit
+          end
+        end
+      rescue Timeout::Error => e
+      rescue Exception => e
+        raise
+      ensure
+        unsubscribe("discovery", :reply)
+      end
+
+      hosts.sort
+    end
+
+    # Send a request, performs the passed block for each response
+    #
+    # times = req("status", "mcollectived", options, client) {|resp|
+    #   pp resp
+    # }
+    #
+    # It returns a hash of times and timeouts for discovery and total run is taken from the options
+    # hash which in turn is generally built using MCollective::Optionparser
+    def req(body, agent=nil, options=false, waitfor=0)
+      if body.is_a?(Message)
+        agent = body.agent
+        options = body.options
+        waitfor = body.discovered_hosts.size || 0
+      end
+
+      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
+
+      options = @options unless options
+
+      STDOUT.sync = true
+
+      hosts_responded = 0
+
+      begin
+        Timeout.timeout(options[:timeout]) do
+          reqid = sendreq(body, agent, options[:filter])
+
+          loop do
+            resp = receive(reqid)
+
+            hosts_responded += 1
+
+            yield(resp.payload)
+
+            break if (waitfor != 0 && hosts_responded >= waitfor)
+          end
+        end
+      rescue Interrupt => e
+      rescue Timeout::Error => e
+      ensure
+        unsubscribe(agent, :reply)
+      end
+
+      stat[:totaltime] = Time.now.to_f - stat[:starttime]
+      stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
+      stat[:responses] = hosts_responded
+      stat[:noresponsefrom] = []
+
+      @stats = stat
+      return stat
+    end
+
+    # Performs a discovery and then send a request, performs the passed block for each response
+    #
+    #    times = discovered_req("status", "mcollectived", options, client) {|resp|
+    #       pp resp
+    #    }
+    #
+    # It returns a hash of times and timeouts for discovery and total run is taken from the options
+    # hash which in turn is generally built using MCollective::Optionparser
+    def discovered_req(body, agent, options=false)
+      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
+
+      options = @options unless options
+
+      STDOUT.sync = true
+
+      print("Determining the amount of hosts matching filter for #{options[:disctimeout]} seconds .... ")
+
+      begin
+        discovered_hosts = discover(options[:filter], options[:disctimeout])
+        discovered = discovered_hosts.size
+        hosts_responded = []
+        hosts_not_responded = discovered_hosts
+
+        stat[:discoverytime] = Time.now.to_f - stat[:starttime]
+
+        puts("#{discovered}\n\n")
+      rescue Interrupt
+        puts("Discovery interrupted.")
+        exit!
+      end
+
+      raise("No matching clients found") if discovered == 0
+
+      begin
+        Timeout.timeout(options[:timeout]) do
+          reqid = sendreq(body, agent, options[:filter])
+
+          (1..discovered).each do |c|
+            resp = receive(reqid)
+
+            hosts_responded << resp.payload[:senderid]
+            hosts_not_responded.delete(resp.payload[:senderid]) if hosts_not_responded.include?(resp.payload[:senderid])
+
+            yield(resp.payload)
+          end
+        end
+      rescue Interrupt => e
+      rescue Timeout::Error => e
+      end
+
+      stat[:totaltime] = Time.now.to_f - stat[:starttime]
+      stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
+      stat[:responses] = hosts_responded.size
+      stat[:responsesfrom] = hosts_responded
+      stat[:noresponsefrom] = hosts_not_responded
+      stat[:discovered] = discovered
+
+      @stats = stat
+      return stat
+    end
+
+    # Prints out the stats returns from req and discovered_req in a nice way
+    def display_stats(stats, options=false, caption="stomp call summary")
+      options = @options unless options
+
+      if options[:verbose]
+        puts("\n---- #{caption} ----")
+
+        if stats[:discovered]
+          puts("           Nodes: #{stats[:discovered]} / #{stats[:responses]}")
+        else
+          puts("           Nodes: #{stats[:responses]}")
+        end
+
+        printf("      Start Time: %s\n", Time.at(stats[:starttime]))
+        printf("  Discovery Time: %.2fms\n", stats[:discoverytime] * 1000)
+        printf("      Agent Time: %.2fms\n", stats[:blocktime] * 1000)
+        printf("      Total Time: %.2fms\n", stats[:totaltime] * 1000)
+
+      else
+        if stats[:discovered]
+          printf("\nFinished processing %d / %d hosts in %.2f ms\n\n", stats[:responses], stats[:discovered], stats[:blocktime] * 1000)
+        else
+          printf("\nFinished processing %d hosts in %.2f ms\n\n", stats[:responses], stats[:blocktime] * 1000)
+        end
+      end
+
+      if stats[:noresponsefrom].size > 0
+        puts("\nNo response from:\n")
+
+        stats[:noresponsefrom].each do |c|
+          puts if c % 4 == 1
+          printf("%30s", c)
+        end
+
+        puts
+      end
+    end
+  end
+end

data/lib/mcollective/config.rb

@@ -0,0 +1,202 @@
+module MCollective
+  # A pretty sucky config class, ripe for refactoring/improving
+  class Config
+    include Singleton
+
+    attr_reader :topicprefix, :daemonize, :pluginconf, :libdir, :configured,
+    :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility, :identity,
+    :daemonize, :connector, :securityprovider, :factsource, :registration,
+    :registerinterval, :topicsep, :classesfile, :rpcauditprovider, :rpcaudit,
+    :configdir, :rpcauthprovider, :rpcauthorization, :color, :configfile,
+    :rpchelptemplate, :rpclimitmethod, :logger_type, :fact_cache_time,
+    :collectives, :main_collective, :ssl_cipher, :registration_collective,
+    :direct_addressing, :direct_addressing_threshold, :queueprefix, :ttl
+
+    def initialize
+      @configured = false
+    end
+
+    def loadconfig(configfile)
+      set_config_defaults(configfile)
+
+      if File.exists?(configfile)
+        File.open(configfile, "r").each do |line|
+
+          # strip blank spaces, tabs etc off the end of all lines
+          line.gsub!(/\s*$/, "")
+
+          unless line =~ /^#|^$/
+            if (line =~ /(.+?)\s*=\s*(.+)/)
+              key = $1
+              val = $2
+
+              case key
+                when "topicsep"
+                  @topicsep = val
+                when "registration"
+                  @registration = val.capitalize
+                when "registration_collective"
+                  @registration_collective = val
+                when "registerinterval"
+                  @registerinterval = val.to_i
+                when "collectives"
+                  @collectives = val.split(",").map {|c| c.strip}
+                when "main_collective"
+                  @main_collective = val
+                when "topicprefix"
+                  @topicprefix = val
+                when "queueprefix"
+                  @queueprefix = val
+                when "logfile"
+                  @logfile = val
+                when "keeplogs"
+                  @keeplogs = val.to_i
+                when "max_log_size"
+                  @max_log_size = val.to_i
+                when "loglevel"
+                  @loglevel = val
+                when "logfacility"
+                  @logfacility = val
+                when "libdir"
+                  paths = val.split(File::PATH_SEPARATOR)
+                  paths.each do |path|
+                    @libdir << path
+                    unless $LOAD_PATH.include?(path)
+                      $LOAD_PATH << path
+                    end
+                  end
+                when "identity"
+                  @identity = val
+                when "direct_addressing"
+                  val =~ /^1|y/i ? @direct_addressing = true : @direct_addressing = false
+                when "direct_addressing_threshold"
+                  @direct_addressing_threshold = val.to_i
+                when "color"
+                  val =~ /^1|y/i ? @color = true : @color = false
+                when "daemonize"
+                  val =~ /^1|y/i ? @daemonize = true : @daemonize = false
+                when "securityprovider"
+                  @securityprovider = val.capitalize
+                when "factsource"
+                  @factsource = val.capitalize
+                when "connector"
+                  @connector = val.capitalize
+                when "classesfile"
+                  @classesfile = val
+                when /^plugin.(.+)$/
+                  @pluginconf[$1] = val
+                when "rpcaudit"
+                  val =~ /^1|y/i ? @rpcaudit = true : @rpcaudit = false
+                when "rpcauditprovider"
+                  @rpcauditprovider = val.capitalize
+                when "rpcauthorization"
+                  val =~ /^1|y/i ? @rpcauthorization = true : @rpcauthorization = false
+                when "rpcauthprovider"
+                  @rpcauthprovider = val.capitalize
+                when "rpchelptemplate"
+                  @rpchelptemplate = val
+                when "rpclimitmethod"
+                  @rpclimitmethod = val.to_sym
+                when "logger_type"
+                  @logger_type = val
+                when "fact_cache_time"
+                  @fact_cache_time = val.to_i
+                when "ssl_cipher"
+                  @ssl_cipher = val
+                when "ttl"
+                  @ttl = val.to_i
+                else
+                  raise("Unknown config parameter #{key}")
+              end
+            end
+          end
+        end
+
+        read_plugin_config_dir("#{@configdir}/plugin.d")
+
+        raise 'Identities can only match /\w\.\-/' unless @identity.match(/^[\w\.\-]+$/)
+
+        @configured = true
+
+        @libdir.each {|dir| Log.warn("Cannot find libdir: #{dir}") unless File.directory?(dir)}
+
+        if @logger_type == "syslog"
+          raise "The sylog logger is not usable on the Windows platform" if Util.windows?
+        end
+
+        PluginManager.loadclass("Mcollective::Facts::#{@factsource}_facts")
+        PluginManager.loadclass("Mcollective::Connector::#{@connector}")
+        PluginManager.loadclass("Mcollective::Security::#{@securityprovider}")
+        PluginManager.loadclass("Mcollective::Registration::#{@registration}")
+        PluginManager.loadclass("Mcollective::Audit::#{@rpcauditprovider}") if @rpcaudit
+        PluginManager << {:type => "global_stats", :class => RunnerStats.new}
+      else
+        raise("Cannot find config file '#{configfile}'")
+      end
+    end
+
+    def set_config_defaults(configfile)
+      @stomp = Hash.new
+      @subscribe = Array.new
+      @pluginconf = Hash.new
+      @connector = "Stomp"
+      @securityprovider = "Psk"
+      @factsource = "Yaml"
+      @identity = Socket.gethostname
+      @registration = "Agentlist"
+      @registerinterval = 0
+      @registration_collective = nil
+      @topicsep = "."
+      @topicprefix = "/topic/"
+      @queueprefix = "/queue/"
+      @classesfile = "/var/lib/puppet/state/classes.txt"
+      @rpcaudit = false
+      @rpcauditprovider = ""
+      @rpcauthorization = false
+      @rpcauthprovider = ""
+      @configdir = File.dirname(configfile)
+      @color = !Util.windows?
+      @configfile = configfile
+      @logger_type = "file"
+      @keeplogs = 5
+      @max_log_size = 2097152
+      @rpclimitmethod = :first
+      @libdir = Array.new
+      @fact_cache_time = 300
+      @loglevel = "info"
+      @logfacility = "user"
+      @collectives = ["mcollective"]
+      @main_collective = @collectives.first
+      @ssl_cipher = "aes-256-cbc"
+      @direct_addressing = false
+      @direct_addressing_threshold = 10
+      @ttl = 60
+
+      # look in the config dir for the template so users can provide their own and windows
+      # with odd paths will just work more often, but fall back to old behavior if it does
+      # not exist
+      @rpchelptemplate = File.join(File.dirname(configfile), "rpc-help.erb")
+      @rpchelptemplate = "/etc/mcollective/rpc-help.erb" unless File.exists?(@rpchelptemplate)
+    end
+
+    def read_plugin_config_dir(dir)
+      return unless File.directory?(dir)
+
+      Dir.new(dir).each do |pluginconfigfile|
+        next unless pluginconfigfile =~ /^([\w]+).cfg$/
+
+        plugin = $1
+        File.open("#{dir}/#{pluginconfigfile}", "r").each do |line|
+          # strip blank lines
+          line.gsub!(/\s*$/, "")
+          next if line =~ /^#|^$/
+          if (line =~ /(.+?)\s*=\s*(.+)/)
+            key = $1
+            val = $2
+            @pluginconf["#{plugin}.#{key}"] = val
+          end
+        end
+      end
+    end
+  end
+end