mcollective-client 2.0.0 → 2.2.0

data/lib/mcollective/client.rb

@@ -1,7 +1,7 @@
 module MCollective
   # Helpers for writing clients that can talk to agents, do discovery and so forth
   class Client
-    attr_accessor :options, :stats
+    attr_accessor :options, :stats, :discoverer
 
     def initialize(configfile)
       @config = Config.instance
@@ -14,6 +14,7 @@ module MCollective
       @options = nil
       @subscriptions = {}
 
+      @discoverer = Discovery.new(self)
       @connection.connect
     end
 
@@ -49,7 +50,7 @@ module MCollective
 
       Log.debug("Sending request #{request.requestid} to the #{request.agent} agent with ttl #{request.ttl} in collective #{request.collective}")
 
-      subscribe(agent, :reply)
+      subscribe(agent, :reply) unless request.reply_to
 
       request.publish
 
@@ -112,30 +113,7 @@ module MCollective
     # of the discovery being cancelled soon as it reached the
     # requested limit of hosts
     def discover(filter, timeout, limit=0)
-      raise "Limit has to be an integer" unless limit.is_a?(Fixnum)
-
-      begin
-        hosts = []
-        Timeout.timeout(timeout) do
-          reqid = sendreq("ping", "discovery", filter)
-          Log.debug("Waiting #{timeout} seconds for discovery replies to request #{reqid}")
-
-          loop do
-            reply = receive(reqid)
-            Log.debug("Got discovery reply from #{reply.payload[:senderid]}")
-            hosts << reply.payload[:senderid]
-
-            return hosts if limit > 0 && hosts.size == limit
-          end
-        end
-      rescue Timeout::Error => e
-      rescue Exception => e
-        raise
-      ensure
-        unsubscribe("discovery", :reply)
-      end
-
-      hosts.sort
+      discovered = @discoverer.discover(filter, timeout, limit)
     end
 
     # Send a request, performs the passed block for each response
@@ -155,15 +133,16 @@ module MCollective
 
       stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
 
-      options = @options unless options
+      timeout = @discoverer.discovery_timeout(@options[:timeout], @options[:filter])
 
       STDOUT.sync = true
 
       hosts_responded = 0
+      reqid = nil
 
       begin
-        Timeout.timeout(options[:timeout]) do
-          reqid = sendreq(body, agent, options[:filter])
+        Timeout.timeout(timeout) do
+          reqid = sendreq(body, agent, @options[:filter])
 
           loop do
             resp = receive(reqid)
@@ -185,70 +164,14 @@ module MCollective
       stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
       stat[:responses] = hosts_responded
       stat[:noresponsefrom] = []
+      stat[:requestid] = reqid
 
       @stats = stat
       return stat
     end
 
-    # Performs a discovery and then send a request, performs the passed block for each response
-    #
-    #    times = discovered_req("status", "mcollectived", options, client) {|resp|
-    #       pp resp
-    #    }
-    #
-    # It returns a hash of times and timeouts for discovery and total run is taken from the options
-    # hash which in turn is generally built using MCollective::Optionparser
     def discovered_req(body, agent, options=false)
-      stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
-
-      options = @options unless options
-
-      STDOUT.sync = true
-
-      print("Determining the amount of hosts matching filter for #{options[:disctimeout]} seconds .... ")
-
-      begin
-        discovered_hosts = discover(options[:filter], options[:disctimeout])
-        discovered = discovered_hosts.size
-        hosts_responded = []
-        hosts_not_responded = discovered_hosts
-
-        stat[:discoverytime] = Time.now.to_f - stat[:starttime]
-
-        puts("#{discovered}\n\n")
-      rescue Interrupt
-        puts("Discovery interrupted.")
-        exit!
-      end
-
-      raise("No matching clients found") if discovered == 0
-
-      begin
-        Timeout.timeout(options[:timeout]) do
-          reqid = sendreq(body, agent, options[:filter])
-
-          (1..discovered).each do |c|
-            resp = receive(reqid)
-
-            hosts_responded << resp.payload[:senderid]
-            hosts_not_responded.delete(resp.payload[:senderid]) if hosts_not_responded.include?(resp.payload[:senderid])
-
-            yield(resp.payload)
-          end
-        end
-      rescue Interrupt => e
-      rescue Timeout::Error => e
-      end
-
-      stat[:totaltime] = Time.now.to_f - stat[:starttime]
-      stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
-      stat[:responses] = hosts_responded.size
-      stat[:responsesfrom] = hosts_responded
-      stat[:noresponsefrom] = hosts_not_responded
-      stat[:discovered] = discovered
-
-      @stats = stat
-      return stat
+      raise "Client#discovered_req has been removed, please port your agent and client to the SimpleRPC framework"
     end
 
     # Prints out the stats returns from req and discovered_req in a nice way

data/lib/mcollective/config.rb

@@ -3,14 +3,18 @@ module MCollective
   class Config
     include Singleton
 
-    attr_reader :topicprefix, :daemonize, :pluginconf, :libdir, :configured,
-    :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility, :identity,
-    :daemonize, :connector, :securityprovider, :factsource, :registration,
-    :registerinterval, :topicsep, :classesfile, :rpcauditprovider, :rpcaudit,
-    :configdir, :rpcauthprovider, :rpcauthorization, :color, :configfile,
-    :rpchelptemplate, :rpclimitmethod, :logger_type, :fact_cache_time,
-    :collectives, :main_collective, :ssl_cipher, :registration_collective,
-    :direct_addressing, :direct_addressing_threshold, :queueprefix, :ttl
+    attr_accessor :mode
+
+    attr_reader :topicprefix, :daemonize, :pluginconf, :libdir, :configured
+    attr_reader :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility
+    attr_reader :identity, :daemonize, :connector, :securityprovider, :factsource
+    attr_reader :registration, :registerinterval, :topicsep, :classesfile
+    attr_reader :rpcauditprovider, :rpcaudit, :configdir, :rpcauthprovider
+    attr_reader :rpcauthorization, :color, :configfile, :rpchelptemplate
+    attr_reader :rpclimitmethod, :logger_type, :fact_cache_time, :collectives
+    attr_reader :main_collective, :ssl_cipher, :registration_collective
+    attr_reader :direct_addressing, :direct_addressing_threshold, :ttl, :helptemplatedir
+    attr_reader :queueprefix, :default_discovery_method, :default_discovery_options
 
     def initialize
       @configured = false
@@ -105,6 +109,12 @@ module MCollective
                   @ssl_cipher = val
                 when "ttl"
                   @ttl = val.to_i
+                when "helptemplatedir"
+                  @helptemplatedir = val
+                when "default_discovery_options"
+                  @default_discovery_options << val
+                when "default_discovery_method"
+                  @default_discovery_method = val
                 else
                   raise("Unknown config parameter #{key}")
               end
@@ -170,13 +180,17 @@ module MCollective
       @ssl_cipher = "aes-256-cbc"
       @direct_addressing = false
       @direct_addressing_threshold = 10
+      @default_discovery_method = "mc"
+      @default_discovery_options = []
       @ttl = 60
+      @mode = :client
 
       # look in the config dir for the template so users can provide their own and windows
       # with odd paths will just work more often, but fall back to old behavior if it does
       # not exist
       @rpchelptemplate = File.join(File.dirname(configfile), "rpc-help.erb")
       @rpchelptemplate = "/etc/mcollective/rpc-help.erb" unless File.exists?(@rpchelptemplate)
+      @helptemplatedir = File.dirname(@rpchelptemplate)
     end
 
     def read_plugin_config_dir(dir)

data/lib/mcollective/data.rb

@@ -0,0 +1,87 @@
+module MCollective
+  module Data
+    autoload :Base, "mcollective/data/base"
+    autoload :Result, "mcollective/data/result"
+
+    def self.load_data_sources
+      PluginManager.find_and_load("data")
+
+      PluginManager.grep(/_data$/).each do |plugin|
+        begin
+          unless PluginManager[plugin].class.activate?
+            Log.debug("Disabling data plugin %s due to plugin activation policy" % plugin)
+            PluginManager.delete(plugin)
+          end
+        rescue Exception => e
+          Log.debug("Disabling data plugin %s due to exception #{e.class}: #{e}" % plugin)
+          PluginManager.delete(plugin)
+        end
+      end
+    end
+
+    def self.pluginname(plugin)
+      plugin.to_s =~ /_data$/i ? plugin.to_s.downcase : "%s_data" % plugin.to_s.downcase
+    end
+
+    def self.[](plugin)
+      PluginManager[pluginname(plugin)]
+    end
+
+    # Data.package("httpd").architecture
+    def self.method_missing(method, *args)
+      super unless PluginManager.include?(pluginname(method))
+
+      PluginManager[pluginname(method)].lookup(args.first)
+    end
+
+    def self.ddl(plugin)
+      DDL.new(pluginname(plugin), :data)
+    end
+
+    def self.ddl_validate(ddl, argument)
+      name = ddl.meta[:name]
+      query = ddl.entities[:data]
+
+      raise DDLValidationError, "No dataquery has been defined in the DDL for data plugin #{name}" unless query
+
+      input = query[:input]
+      output = query[:output]
+
+      raise DDLValidationError, "No :query input has been defined in the DDL for data plugin #{name}" unless input[:query]
+      raise DDLValidationError, "No output has been defined in the DDL for data plugin #{name}" if output.keys.empty?
+
+      return true if argument.nil? && input[:query][:optional]
+
+      ddl.validate_input_argument(input, :query, argument)
+    end
+
+    def self.ddl_has_output?(ddl, output)
+      ddl.entities[:data][:output].include?(output.to_sym) rescue false
+    end
+
+    # For an input where the DDL requests a boolean or some number
+    # this will convert the input to the right type where possible
+    # else just returns the origin input unedited
+    #
+    # if anything here goes wrong just return the input value
+    # this is not really the end of the world or anything since
+    # all that will happen is that DDL validation will fail and
+    # the user will get an error, no need to be too defensive here
+    def self.ddl_transform_input(ddl, input)
+      begin
+        type = ddl.entities[:data][:input][:query][:type]
+
+        case type
+          when :boolean
+            return DDL.string_to_boolean(input)
+
+          when :number, :integer, :float
+            return DDL.string_to_number(input)
+        end
+      rescue
+      end
+
+      return input
+    end
+  end
+end

data/lib/mcollective/data/base.rb

@@ -0,0 +1,67 @@
+module MCollective
+  module Data
+    class Base
+      attr_reader :name, :result, :ddl, :timeout
+
+      # Register plugins that inherits base
+      def self.inherited(klass)
+        type = klass.to_s.split("::").last.downcase
+
+        PluginManager << {:type => type, :class => klass.to_s, :single_instance => false}
+      end
+
+      def initialize
+        @name = self.class.to_s.split("::").last.downcase
+        @ddl = DDL.new(@name, :data)
+        @result = Result.new
+        @timeout = @ddl.meta[:timeout] || 1
+
+        startup_hook
+      end
+
+      def lookup(what)
+        ddl_validate(what)
+
+        Log.debug("Doing data query %s for '%s'" % [ @name, what ])
+
+        Timeout::timeout(@timeout) do
+          query_data(what)
+        end
+
+        @result
+      rescue Timeout::Error
+        # Timeout::Error is a inherited from Interrupt which seems a really
+        # strange choice, making it an equivelant of ^C and such.  Catch it
+        # and raise something less critical that will not the runner to just
+        # give up the ghost
+        msg = "Data plugin %s timed out on query '%s'" % [@name, what]
+        Log.error(msg)
+        raise MsgTTLExpired, msg
+      end
+
+      def self.query(&block)
+        self.module_eval { define_method("query_data", &block) }
+      end
+
+      def ddl_validate(what)
+        Data.ddl_validate(@ddl, what)
+      end
+
+      # activate_when do
+      #    file.exist?("/usr/bin/puppet")
+      # end
+      def self.activate_when(&block)
+        (class << self; self; end).instance_eval do
+          define_method("activate?", &block)
+        end
+      end
+
+      # Always be active unless a specific block is given with activate_when
+      def self.activate?
+        return true
+      end
+
+      def startup_hook;end
+    end
+  end
+end

data/lib/mcollective/data/result.rb

@@ -0,0 +1,40 @@
+module MCollective
+  module Data
+    class Result
+      # remove some methods that might clash with commonly
+      # used return data to improve the effectiveness of the
+      # method_missing lookup strategy
+      undef :type if method_defined?(:type)
+
+      def initialize
+        @data = {}
+      end
+
+      def include?(key)
+        @data.include?(key.to_sym)
+      end
+
+      def [](key)
+        @data[key.to_sym]
+      end
+
+      def []=(key, val)
+        raise "Can only store String, Integer, Float or Boolean data but got #{val.class} for key #{key}" unless [String, Fixnum, Float, TrueClass, FalseClass].include?(val.class)
+
+        @data[key.to_sym] = val
+      end
+
+      def keys
+        @data.keys
+      end
+
+      def method_missing(method, *args)
+        key = method.to_sym
+
+        raise NameError, "undefined local variable or method `%s'" % key unless include?(key)
+
+        @data[key]
+      end
+    end
+  end
+end

data/lib/mcollective/ddl.rb

@@ -0,0 +1,113 @@
+module MCollective
+  # A set of classes that helps create data description language files
+  # for plugins.  You can define meta data, actions, input and output
+  # describing the behavior of your agent or other plugins
+  #
+  # DDL files are used for input validation, constructing outputs,
+  # producing online help, informing the various display routines and
+  # so forth.
+  #
+  # A sample DDL for an agent be seen below, you'd put this in your agent
+  # dir as <agent name>.ddl
+  #
+  #    metadata :name        => "SimpleRPC Service Agent",
+  #             :description => "Agent to manage services using the Puppet service provider",
+  #             :author      => "R.I.Pienaar",
+  #             :license     => "GPLv2",
+  #             :version     => "1.1",
+  #             :url         => "http://mcollective-plugins.googlecode.com/",
+  #             :timeout     => 60
+  #
+  #    action "status", :description => "Gets the status of a service" do
+  #       display :always
+  #
+  #       input :service,
+  #             :prompt      => "Service Name",
+  #             :description => "The service to get the status for",
+  #             :type        => :string,
+  #             :validation  => '^[a-zA-Z\-_\d]+$',
+  #             :optional    => true,
+  #             :maxlength   => 30
+  #
+  #       output :status,
+  #              :description => "The status of service",
+  #              :display_as  => "Service Status"
+  #   end
+  #
+  # There are now many types of DDL and ultimately all pugins should have
+  # DDL files.  The code is organized so that any plugin type will magically
+  # just work - they will be an instane of Base which has #metadata and a few
+  # common cases.
+  #
+  # For plugin types that require more specific behaviors they can just add a
+  # class here that inherits from Base and add their specific behavior.
+  #
+  # Base defines a specific behavior for input, output and metadata which we'd
+  # like to keep standard across plugin types so do not completely override the
+  # behavior of input.  The methods are written that they will gladly store extra
+  # content though so you add, do not remove.  See the AgentDDL class for an example
+  # where agents want a :required argument to be always set.
+  module DDL
+    autoload :Base, "mcollective/ddl/base"
+    autoload :AgentDDL, "mcollective/ddl/agentddl"
+    autoload :DataDDL, "mcollective/ddl/dataddl"
+    autoload :DiscoveryDDL, "mcollective/ddl/discoveryddl"
+
+    # There used to be only one big nasty DDL class with a bunch of mashed
+    # together behaviors.  It's been around for ages and we would rather not
+    # ask all the users to change their DDL.new calls to some other factory
+    # method that would have this exact same behavior.
+    #
+    # So we override the behavior of #new which is a hugely sucky thing to do
+    # but ultimately it's what would be least disrupting to code out there
+    # today.  We did though change DDL to a module to make it possibly a
+    # little less suprising, possibly.
+    def self.new(*args, &blk)
+      load_and_cache(*args)
+    end
+
+    def self.load_and_cache(*args)
+      Cache.setup(:ddl, 300)
+
+      plugin = args.first
+      args.size > 1 ? type = args[1].to_s : type = "agent"
+      path = "%s/%s" % [type, plugin]
+
+      begin
+        ddl = Cache.read(:ddl, path)
+      rescue
+        begin
+          klass = DDL.const_get("%sDDL" % type.capitalize)
+        rescue NameError
+          klass = Base
+        end
+
+        ddl = Cache.write(:ddl, path, klass.new(*args))
+      end
+
+      return ddl
+    end
+
+    # As we're taking arguments on the command line we need a
+    # way to input booleans, true on the cli is a string so this
+    # method will take the ddl, find all arguments that are supposed
+    # to be boolean and if they are the strings "true"/"yes" or "false"/"no"
+    # turn them into the matching boolean
+    def self.string_to_boolean(val)
+      return true if ["true", "t", "yes", "y", "1"].include?(val.downcase)
+      return false if ["false", "f", "no", "n", "0"].include?(val.downcase)
+
+      raise "#{val} does not look like a boolean argument"
+    end
+
+    # a generic string to number function, if a number looks like a float
+    # it turns it into a float else an int.  This is naive but should be sufficient
+    # for numbers typed on the cli in most cases
+    def self.string_to_number(val)
+      return val.to_f if val =~ /^\d+\.\d+$/
+      return val.to_i if val =~ /^\d+$/
+
+      raise "#{val} does not look like a number"
+    end
+  end
+end