choria-mcorpc-support 0.0.1

data/lib/mcollective/application/rpc.rb

@@ -0,0 +1,111 @@
+class MCollective::Application::Rpc<MCollective::Application
+  description "Generic RPC agent client application"
+
+  usage "mco rpc [options] [filters] --agent <agent> --action <action> [--argument <key=val> --argument ...]"
+  usage "mco rpc [options] [filters] <agent> <action> [<key=val> <key=val> ...]"
+
+  option :show_results,
+         :description    => "Do not process results, just send request",
+         :arguments      => ["--no-results", "--nr"],
+         :default        => true,
+         :type           => :bool
+
+  option :agent,
+         :description    => "Agent to call",
+         :arguments      => ["-a", "--agent AGENT"]
+
+  option :action,
+         :description    => "Action to call",
+         :arguments      => ["--action ACTION"]
+
+  option :arguments,
+         :description    => "Arguments to pass to agent",
+         :arguments      => ["--arg", "--argument ARGUMENT"],
+         :type           => :array,
+         :default        => [],
+         :validate       => Proc.new {|val| val.match(/^(.+?)=(.+)$/) ? true : "Could not parse --arg #{val} should be of the form key=val" }
+
+  def post_option_parser(configuration)
+    # handle the alternative format that optparse cant parse
+    unless (configuration.include?(:agent) && configuration.include?(:action))
+      if ARGV.length >= 2
+        configuration[:agent] = ARGV[0]
+        ARGV.delete_at(0)
+
+        configuration[:action] = ARGV[0]
+        ARGV.delete_at(0)
+
+        ARGV.each do |v|
+          if v =~ /^(.+?)=(.+)$/
+            configuration[:arguments] = [] unless configuration.include?(:arguments)
+            configuration[:arguments] << v
+          else
+            STDERR.puts("Could not parse --arg #{v}")
+            exit(1)
+          end
+        end
+      else
+        STDERR.puts("No agent, action and arguments specified")
+        exit(1)
+      end
+    end
+
+    # convert arguments to symbols for keys to comply with simplerpc conventions
+    args = configuration[:arguments].clone
+    configuration[:arguments] = {}
+
+    args.each do |v|
+      if v =~ /^(.+?)=(.+)$/
+        configuration[:arguments][$1.to_sym] = $2
+      end
+    end
+  end
+
+  def string_to_ddl_type(arguments, ddl)
+    return if ddl.empty?
+
+    arguments.keys.each do |key|
+      if ddl[:input].keys.include?(key)
+        case ddl[:input][key][:type]
+          when :boolean
+            arguments[key] = MCollective::DDL.string_to_boolean(arguments[key])
+
+          when :number, :integer, :float
+            arguments[key] = MCollective::DDL.string_to_number(arguments[key])
+        end
+      end
+    end
+  end
+
+  def main
+    mc = rpcclient(configuration[:agent])
+
+    mc.agent_filter(configuration[:agent])
+
+    string_to_ddl_type(configuration[:arguments], mc.ddl.action_interface(configuration[:action])) if mc.ddl
+
+    mc.validate_request(configuration[:action], configuration[:arguments])
+
+    if mc.reply_to
+      configuration[:arguments][:process_results] = true
+
+      puts "Request sent with id: " + mc.send(configuration[:action], configuration[:arguments]) + " replies to #{mc.reply_to}"
+    elsif !configuration[:show_results]
+      configuration[:arguments][:process_results] = false
+
+      puts "Request sent with id: " + mc.send(configuration[:action], configuration[:arguments])
+    else
+      discover_args = {:verbose => true}
+
+      mc.detect_and_set_stdin_discovery
+
+      mc.discover discover_args
+
+      printrpc mc.send(configuration[:action], configuration[:arguments])
+
+      printrpcstats :summarize => true, :caption => "#{configuration[:agent]}##{configuration[:action]} call stats" if mc.discover.size > 0
+
+      halt mc.stats
+    end
+  end
+end

data/lib/mcollective/applications.rb

@@ -0,0 +1,134 @@
+module MCollective
+  class Applications
+    def self.[](appname)
+      load_application(appname)
+      PluginManager["#{appname}_application"]
+    end
+
+    def self.run(appname)
+      load_config
+
+      begin
+        load_application(appname)
+      rescue Exception => e
+        e.backtrace.first << Util.colorize(:red, "  <----")
+        STDERR.puts "Application '#{appname}' failed to load:"
+        STDERR.puts
+        STDERR.puts Util.colorize(:red, "   #{e} (#{e.class})")
+        STDERR.puts
+        STDERR.puts "       %s" % [e.backtrace.join("\n       ")]
+        exit 1
+      end
+
+      PluginManager["#{appname}_application"].run
+    end
+
+    def self.load_application(appname)
+      return if PluginManager.include?("#{appname}_application")
+
+      load_config
+
+      PluginManager.loadclass "MCollective::Application::#{appname.capitalize}"
+      PluginManager << {:type => "#{appname}_application", :class => "MCollective::Application::#{appname.capitalize}"}
+    end
+
+    # Returns an array of applications found in the lib dirs
+    def self.list
+      load_config
+
+      PluginManager.find("application")
+    rescue SystemExit
+      exit 1
+    rescue Exception => e
+      STDERR.puts "Failed to generate application list: #{e.class}: #{e}"
+      exit 1
+    end
+
+    # Filters a string of opts out using Shellwords
+    # keeping only things related to --config and -c
+    def self.filter_extra_options(opts)
+      res = ""
+      words = Shellwords.shellwords(opts)
+      words.each_with_index do |word,idx|
+        if word == "-c"
+          return "--config=#{words[idx + 1]}"
+        elsif word == "--config"
+          return "--config=#{words[idx + 1]}"
+        elsif word =~ /\-c=/
+          return word
+        elsif word =~ /\-\-config=/
+          return word
+        end
+      end
+
+      return ""
+    end
+
+    # We need to know the config file in order to know the libdir
+    # so that we can find applications.
+    #
+    # The problem is the CLI might be stuffed with options only the
+    # app in the libdir might understand so we have a chicken and
+    # egg situation.
+    #
+    # We're parsing and filtering MCOLLECTIVE_EXTRA_OPTS removing
+    # all but config related options and parsing the options looking
+    # just for the config file.
+    #
+    # We're handling failures gracefully and finally restoring the
+    # ARG and MCOLLECTIVE_EXTRA_OPTS to the state they were before
+    # we started parsing.
+    #
+    # This is mostly a hack, when we're redoing how config works
+    # this stuff should be made less sucky
+    def self.load_config
+      return if Config.instance.configured
+
+      original_argv = ARGV.clone
+      original_extra_opts = ENV["MCOLLECTIVE_EXTRA_OPTS"].clone rescue nil
+      configfile = nil
+
+      parser = OptionParser.new
+      parser.on("--config CONFIG", "-c", "Config file") do |f|
+        configfile = f
+      end
+
+      parser.program_name = $0
+
+      parser.on("--help")
+
+      # avoid option parsers own internal version handling that sux
+      parser.on("-v", "--verbose")
+
+      if original_extra_opts
+        begin
+          # optparse will parse the whole ENV in one go and refuse
+          # to play along with the retry trick I do below so in
+          # order to handle unknown options properly I parse out
+          # only -c and --config deleting everything else and
+          # then restore the environment variable later when I
+          # am done with it
+          ENV["MCOLLECTIVE_EXTRA_OPTS"] = filter_extra_options(ENV["MCOLLECTIVE_EXTRA_OPTS"].clone)
+          parser.environment("MCOLLECTIVE_EXTRA_OPTS")
+        rescue Exception => e
+          Log.error("Failed to parse MCOLLECTIVE_EXTRA_OPTS: #{e}")
+        end
+
+        ENV["MCOLLECTIVE_EXTRA_OPTS"] = original_extra_opts.clone
+      end
+
+      begin
+        parser.parse!
+      rescue OptionParser::InvalidOption => e
+        retry
+      end
+
+      ARGV.clear
+      original_argv.each {|a| ARGV << a}
+
+      configfile = Util.config_file_for_user unless configfile
+
+      Config.instance.loadconfig(configfile)
+    end
+  end
+end

data/lib/mcollective/cache.rb

@@ -0,0 +1,145 @@
+module MCollective
+  # A class to manage a number of named caches.  Each cache can have a unique
+  # timeout for keys in it and each cache has an independent Mutex protecting
+  # access to it.
+  #
+  # This cache is setup early in the process of loading the mcollective
+  # libraries, before any threads are created etc making it suitable as a
+  # cross thread cache or just a store for Mutexes you need to use across
+  # threads like in an Agent or something.
+  #
+  #    # sets up a new cache, noop if it already exist
+  #    Cache.setup(:ddl, 600)
+  #    => true
+  #
+  #    # writes an item to the :ddl cache, this item will
+  #    # be valid on the cache for 600 seconds
+  #    Cache.write(:ddl, :something, "value")
+  #    => "value"
+  #
+  #    # reads from the cache, read failures due to non existing
+  #    # data or expired data will raise an exception
+  #    Cache.read(:ddl, :something)
+  #    => "value"
+  #
+  #    # time left till expiry, raises if nothing is found
+  #    Cache.ttl(:ddl, :something)
+  #    => 500
+  #
+  #    # forcibly evict something from the cache
+  #    Cache.invalidate!(:ddl, :something)
+  #    => "value"
+  #
+  #    # deletes an entire named cache and its mutexes
+  #    Cache.delete!(:ddl)
+  #    => true
+  #
+  #    # you can also just use this cache as a global mutex store
+  #    Cache.setup(:mylock)
+  #
+  #    Cache.synchronize(:mylock) do
+  #      do_something()
+  #    end
+  #
+  module Cache
+    # protects access to @cache_locks and top level @cache
+    @locks_mutex = Mutex.new
+
+    # stores a mutex per named cache
+    @cache_locks = {}
+
+    # the named caches protected by items in @cache_locks
+    @cache = {}
+
+    def self.setup(cache_name, ttl=300)
+      @locks_mutex.synchronize do
+        break if @cache_locks.include?(cache_name)
+
+        @cache_locks[cache_name] = Mutex.new
+
+        @cache_locks[cache_name].synchronize do
+          @cache[cache_name] = {:max_age => Float(ttl)}
+        end
+      end
+
+      true
+    end
+
+    def self.has_cache?(cache_name)
+      @locks_mutex.synchronize do
+        @cache.include?(cache_name)
+      end
+    end
+
+    def self.delete!(cache_name)
+      @locks_mutex.synchronize do
+        raise("No cache called '%s'" % cache_name) unless @cache_locks.include?(cache_name)
+
+        @cache_locks.delete(cache_name)
+        @cache.delete(cache_name)
+      end
+
+      true
+    end
+
+    def self.write(cache_name, key, value)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        @cache[cache_name][key] ||= {}
+        @cache[cache_name][key][:cache_create_time] = Time.now
+        @cache[cache_name][key][:value] = value
+      end
+
+      value
+    end
+
+    def self.read(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      unless ttl(cache_name, key) > 0
+        Log.debug("Cache expired on '%s' key '%s'" % [cache_name, key])
+        raise("Cache for item '%s' on cache '%s' has expired" % [key, cache_name])
+      end
+
+      Log.debug("Cache hit on '%s' key '%s'" % [cache_name, key])
+
+      @cache_locks[cache_name].synchronize do
+        @cache[cache_name][key][:value]
+      end
+    end
+
+    def self.ttl(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        unless @cache[cache_name].include?(key)
+          Log.debug("Cache miss on '%s' key '%s'" % [cache_name, key])
+          raise("No item called '%s' for cache '%s'" % [key, cache_name])
+        end
+
+        @cache[cache_name][:max_age] - (Time.now - @cache[cache_name][key][:cache_create_time])
+      end
+    end
+
+    def self.synchronize(cache_name)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      raise ("No block supplied to synchronize") unless block_given?
+
+      @cache_locks[cache_name].synchronize do
+        yield
+      end
+    end
+
+    def self.invalidate!(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        return false unless @cache[cache_name].include?(key)
+
+        @cache[cache_name].delete(key)
+      end
+    end
+  end
+end

data/lib/mcollective/client.rb

@@ -0,0 +1,353 @@
+module MCollective
+  # Helpers for writing clients that can talk to agents, do discovery and so forth
+  class Client
+    attr_accessor :options, :stats, :discoverer, :connection_timeout
+
+    def initialize(options)
+      @config = Config.instance
+      @options = nil
+
+      if options.is_a?(String)
+        # String is the path to a config file
+        @config.loadconfig(options) unless @config.configured
+      elsif options.is_a?(Hash)
+        @config.loadconfig(options[:config]) unless @config.configured
+        @options = options
+        @connection_timeout = options[:connection_timeout]
+      else
+        raise "Invalid parameter passed to Client constructor. Valid types are Hash or String"
+      end
+
+      @connection_timeout ||= @config.connection_timeout
+
+      @connection = PluginManager["connector_plugin"]
+      @security = PluginManager["security_plugin"]
+
+      @security.initiated_by = :client
+      @subscriptions = {}
+
+      @discoverer = Discovery.new(self)
+
+      # Time box the connection if a timeout has been specified
+      # connection_timeout defaults to nil which means it will try forever if
+      # not specified
+      begin
+        Timeout::timeout(@connection_timeout, ClientTimeoutError) do
+          @connection.connect
+        end
+      rescue ClientTimeoutError => e
+        Log.error("Timeout occured while trying to connect to middleware")
+        raise e
+      end
+    end
+
+    @@request_sequence = 0
+    def self.request_sequence
+      @@request_sequence
+    end
+
+    # Returns the configured main collective if no
+    # specific collective is specified as options
+    def collective
+      if @options[:collective].nil?
+        @config.main_collective
+      else
+        @options[:collective]
+      end
+    end
+
+    # Disconnects cleanly from the middleware
+    def disconnect
+      Log.debug("Disconnecting from the middleware")
+      @connection.disconnect
+    end
+
+    # Sends a request and returns the generated request id, doesn't wait for
+    # responses and doesn't execute any passed in code blocks for responses
+    def sendreq(msg, agent, filter = {})
+      request = createreq(msg, agent, filter)
+      publish(request)
+      request.requestid
+    end
+
+    def createreq(msg, agent, filter ={})
+      if msg.is_a?(Message)
+        request = msg
+        agent = request.agent
+      else
+        ttl = @options[:ttl] || @config.ttl
+        request = Message.new(msg, nil, {:agent => agent, :type => :request, :collective => collective, :filter => filter, :ttl => ttl})
+        request.reply_to = @options[:reply_to] if @options[:reply_to]
+      end
+
+      @@request_sequence += 1
+
+      request.encode!
+      subscribe(agent, :reply) unless request.reply_to
+      request
+    end
+
+    def subscribe(agent, type)
+      unless @subscriptions.include?(agent)
+        subscription = Util.make_subscriptions(agent, type, collective)
+        Log.debug("Subscribing to #{type} target for agent #{agent}")
+
+        Util.subscribe(subscription)
+        @subscriptions[agent] = 1
+      end
+    end
+
+    def unsubscribe(agent, type)
+      if @subscriptions.include?(agent)
+        subscription = Util.make_subscriptions(agent, type, collective)
+        Log.debug("Unsubscribing #{type} target for #{agent}")
+
+        Util.unsubscribe(subscription)
+        @subscriptions.delete(agent)
+      end
+    end
+    # Blocking call that waits for ever for a message to arrive.
+    #
+    # If you give it a requestid this means you've previously send a request
+    # with that ID and now you just want replies that matches that id, in that
+    # case the current connection will just ignore all messages not directed at it
+    # and keep waiting for more till it finds a matching message.
+    def receive(requestid = nil)
+      reply = nil
+
+      begin
+        reply = @connection.receive
+        reply.type = :reply
+        reply.expected_msgid = requestid
+
+        reply.decode!
+
+        unless reply.requestid == requestid
+          raise(MsgDoesNotMatchRequestID, "Message reqid #{reply.requestid} does not match our reqid #{requestid}")
+        end
+
+        Log.debug("Received reply to #{reply.requestid} from #{reply.payload[:senderid]}")
+      rescue SecurityValidationFailed => e
+        Log.warn("Ignoring a message that did not pass security validations")
+        retry
+      rescue MsgDoesNotMatchRequestID => e
+        Log.debug("Ignoring a message for some other client : #{e.message}")
+        retry
+      end
+
+      reply
+    end
+
+    # Performs a discovery of nodes matching the filter passed
+    # returns an array of nodes
+    #
+    # An integer limit can be supplied this will have the effect
+    # of the discovery being cancelled soon as it reached the
+    # requested limit of hosts
+    def discover(filter, timeout, limit=0)
+      @discoverer.discover(filter.merge({'collective' => collective}), timeout, limit)
+    end
+
+    # Send a request, performs the passed block for each response
+    #
+    # times = req("status", "mcollectived", options, client) {|resp|
+    #   pp resp
+    # }
+    #
+    # It returns a hash of times and timeouts for discovery and total run is taken from the options
+    # hash which in turn is generally built using MCollective::Optionparser
+    def req(body, agent=nil, options=false, waitfor=[], &block)
+      if body.is_a?(Message)
+        agent = body.agent
+        waitfor = body.discovered_hosts || []
+        @options = body.options
+      end
+
+      @options = options if options
+      threaded = @options[:threaded]
+      timeout = @discoverer.discovery_timeout(@options[:timeout], @options[:filter])
+      request = createreq(body, agent, @options[:filter])
+      publish_timeout = @options[:publish_timeout] || @config.publish_timeout
+      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
+      STDOUT.sync = true
+      hosts_responded = 0
+
+      begin
+        if threaded
+          hosts_responded = threaded_req(request, publish_timeout, timeout, waitfor, &block)
+        else
+          hosts_responded = unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
+        end
+      rescue Interrupt => e
+      ensure
+        unsubscribe(agent, :reply)
+      end
+
+      return update_stat(stat, hosts_responded, request.requestid)
+    end
+
+    # Starts the client receiver and publisher unthreaded.
+    # This is the default client behaviour.
+    def unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
+      start_publisher(request, publish_timeout)
+      start_receiver(request.requestid, waitfor, timeout, &block)
+    end
+
+    # Starts the client receiver and publisher in threads.
+    # This is activated when the 'threader_client' configuration
+    # option is set.
+    def threaded_req(request, publish_timeout, timeout, waitfor, &block)
+      Log.debug("Starting threaded client")
+      publisher = Thread.new do
+        start_publisher(request, publish_timeout)
+      end
+
+      # When the client is threaded we add the publishing timeout to
+      # the agent timeout so that the receiver doesn't time out before
+      # publishing has finished in cases where publish_timeout >= timeout.
+      total_timeout = publish_timeout + timeout
+      hosts_responded = 0
+
+      receiver = Thread.new do
+        hosts_responded = start_receiver(request.requestid, waitfor, total_timeout, &block)
+      end
+
+      receiver.join
+      hosts_responded
+    end
+
+    # Starts the request publishing routine
+    def start_publisher(request, publish_timeout)
+      Log.debug("Starting publishing with publish timeout of #{publish_timeout}")
+      begin
+        Timeout.timeout(publish_timeout) do
+          publish(request)
+        end
+      rescue Timeout::Error => e
+        Log.warn("Could not publish all messages. Publishing timed out.")
+      end
+    end
+
+    def publish(request)
+      Log.info("Sending request #{request.requestid} for agent '#{request.agent}' with ttl #{request.ttl} in collective '#{request.collective}'")
+      request.publish
+    end
+
+    # Starts the response receiver routine
+    # Expected to return the amount of received responses.
+    def start_receiver(requestid, waitfor, timeout, &block)
+      Log.debug("Starting response receiver with timeout of #{timeout}")
+      hosts_responded = 0
+
+      if (waitfor.is_a?(Array))
+        unfinished = Hash.new(0)
+        waitfor.each {|w| unfinished[w] += 1}
+      else
+        unfinished = []
+      end
+
+      begin
+        Timeout.timeout(timeout) do
+          loop do
+            resp = receive(requestid)
+
+            if block.arity == 2
+              yield resp.payload, resp
+            else
+              yield resp.payload
+            end
+
+            hosts_responded += 1
+
+            if (waitfor.is_a?(Array))
+              sender = resp.payload[:senderid]
+              if unfinished[sender] <= 1
+                unfinished.delete(sender)
+              else
+                unfinished[sender] -= 1
+              end
+
+              break if !waitfor.empty? && unfinished.empty?
+            else
+              break unless waitfor == 0 || hosts_responded < waitfor
+            end
+          end
+        end
+      rescue Timeout::Error => e
+        if waitfor.is_a?(Array)
+          if !unfinished.empty?
+            Log.warn("Could not receive all responses. Did not receive responses from #{unfinished.keys.join(', ')}")
+          end
+        elsif (waitfor > hosts_responded)
+          Log.warn("Could not receive all responses. Expected : #{waitfor}. Received : #{hosts_responded}")
+        end
+      end
+
+      hosts_responded
+    end
+
+    def update_stat(stat, hosts_responded, requestid)
+      stat[:totaltime] = Time.now.to_f - stat[:starttime]
+      stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
+      stat[:responses] = hosts_responded
+      stat[:noresponsefrom] = []
+      stat[:unexpectedresponsefrom] = []
+      stat[:requestid] = requestid
+
+      @stats = stat
+    end
+
+    def discovered_req(body, agent, options=false)
+      raise "Client#discovered_req has been removed, please port your agent and client to the SimpleRPC framework"
+    end
+
+    # Prints out the stats returns from req and discovered_req in a nice way
+    def display_stats(stats, options=false, caption="stomp call summary")
+      options = @options unless options
+
+      if options[:verbose]
+        puts("\n---- #{caption} ----")
+
+        if stats[:discovered]
+          puts("           Nodes: #{stats[:discovered]} / #{stats[:responses]}")
+        else
+          puts("           Nodes: #{stats[:responses]}")
+        end
+
+        printf("      Start Time: %s\n", Time.at(stats[:starttime]))
+        printf("  Discovery Time: %.2fms\n", stats[:discoverytime] * 1000)
+        printf("      Agent Time: %.2fms\n", stats[:blocktime] * 1000)
+        printf("      Total Time: %.2fms\n", stats[:totaltime] * 1000)
+
+      else
+        if stats[:discovered]
+          printf("\nFinished processing %d / %d hosts in %.2f ms\n\n", stats[:responses], stats[:discovered], stats[:blocktime] * 1000)
+        else
+          printf("\nFinished processing %d hosts in %.2f ms\n\n", stats[:responses], stats[:blocktime] * 1000)
+        end
+      end
+
+      if stats[:noresponsefrom].size > 0
+        puts("\nNo response from:\n")
+
+        stats[:noresponsefrom].each do |c|
+          puts if c % 4 == 1
+          printf("%30s", c)
+        end
+
+        puts
+      end
+
+      if stats[:unexpectedresponsefrom].size > 0
+        puts("\nUnexpected response from:\n")
+
+        stats[:unexpectedresponsefrom].each do |c|
+          puts if c % 4 == 1
+          printf("%30s", c)
+        end
+
+        puts
+      end
+    end
+  end
+end