choria-mcorpc-support 0.0.1

data/lib/mcollective/config.rb

@@ -0,0 +1,245 @@
+module MCollective
+  # A pretty sucky config class, ripe for refactoring/improving
+  class Config
+    include Singleton
+
+    attr_accessor :mode
+
+    attr_reader :daemonize, :pluginconf, :configured
+    attr_reader :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility
+    attr_reader :identity, :daemonize, :connector, :securityprovider, :factsource
+    attr_reader :registration, :registerinterval, :classesfile
+    attr_reader :rpcauditprovider, :rpcaudit, :configdir, :rpcauthprovider
+    attr_reader :rpcauthorization, :color, :configfile
+    attr_reader :rpclimitmethod, :logger_type, :fact_cache_time, :collectives
+    attr_reader :main_collective, :ssl_cipher, :registration_collective
+    attr_reader :direct_addressing, :direct_addressing_threshold, :ttl
+    attr_reader :default_discovery_method, :default_discovery_options
+    attr_reader :publish_timeout, :threaded, :soft_shutdown, :activate_agents
+    attr_reader :registration_splay, :discovery_timeout, :soft_shutdown_timeout
+    attr_reader :connection_timeout, :default_batch_size, :default_batch_sleep_time
+
+    def initialize
+      @configured = false
+    end
+
+    def loadconfig(configfile)
+      set_config_defaults(configfile)
+
+      if File.exists?(configfile)
+        libdirs = []
+        File.readlines(configfile).each do |line|
+
+          # strip blank spaces, tabs etc off the end of all lines
+          line.gsub!(/\s*$/, "")
+
+          unless line =~ /^#|^$/
+            if (line =~ /(.+?)\s*=\s*(.+)/)
+              key = $1.strip
+              val = $2
+
+              begin
+                case key
+                when "registration"
+                  @registration = val.capitalize
+                when "registration_collective"
+                  @registration_collective = val
+                when "registerinterval"
+                  @registerinterval = Integer(val)
+                when "registration_splay"
+                  @registration_splay = Util.str_to_bool(val)
+                when "collectives"
+                  @collectives = val.split(",").map {|c| c.strip}
+                when "main_collective"
+                  @main_collective = val
+                when "logfile"
+                  @logfile = val
+                when "keeplogs"
+                  @keeplogs = Integer(val)
+                when "max_log_size"
+                  @max_log_size = Integer(val)
+                when "loglevel"
+                  @loglevel = val
+                when "logfacility"
+                  @logfacility = val
+                when "libdir"
+                  paths = val.split(File::PATH_SEPARATOR)
+                  paths.each do |path|
+                    raise("libdir paths should be absolute paths but '%s' is relative" % path) unless Util.absolute_path?(path)
+
+                    libdirs << path
+                  end
+                when "identity"
+                  @identity = val
+                when "direct_addressing"
+                  @direct_addressing = Util.str_to_bool(val)
+                when "direct_addressing_threshold"
+                  @direct_addressing_threshold = Integer(val)
+                when "color"
+                  @color = Util.str_to_bool(val)
+                when "daemonize"
+                  @daemonize = Util.str_to_bool(val)
+                when "securityprovider"
+                  @securityprovider = val.capitalize
+                when "factsource"
+                  @factsource = val.capitalize
+                when "connector"
+                  @connector = val.capitalize
+                when "classesfile"
+                  @classesfile = val
+                when /^plugin.(.+)$/
+                  @pluginconf[$1] = val
+                when "discovery_timeout"
+                  @discovery_timeout = Integer(val)
+                when "publish_timeout"
+                  @publish_timeout = Integer(val)
+                when "connection_timeout"
+                  @connection_timeout = Integer(val)
+                when "rpcaudit"
+                  @rpcaudit = Util.str_to_bool(val)
+                when "rpcauditprovider"
+                  @rpcauditprovider = val.capitalize
+                when "rpcauthorization"
+                  @rpcauthorization = Util.str_to_bool(val)
+                when "rpcauthprovider"
+                  @rpcauthprovider = val.capitalize
+                when "rpclimitmethod"
+                  @rpclimitmethod = val.to_sym
+                when "logger_type"
+                  @logger_type = val
+                when "fact_cache_time"
+                  @fact_cache_time = Integer(val)
+                when "ssl_cipher"
+                  @ssl_cipher = val
+                when "threaded"
+                  @threaded = Util.str_to_bool(val)
+                when "ttl"
+                  @ttl = Integer(val)
+                when "default_discovery_options"
+                  @default_discovery_options << val
+                when "default_discovery_method"
+                  @default_discovery_method = val
+                when "soft_shutdown"
+                  @soft_shutdown = Util.str_to_bool(val)
+                when "soft_shutdown_timeout"
+                  @soft_shutdown_timeout = Integer(val)
+                when "activate_agents"
+                  @activate_agents = Util.str_to_bool(val)
+                when "default_batch_size"
+                  @default_batch_size = Integer(val)
+                when "default_batch_sleep_time"
+                  @default_batch_sleep_time = Float(val)
+                when "topicprefix", "topicsep", "queueprefix", "rpchelptemplate", "helptemplatedir"
+                  Log.warn("Use of deprecated '#{key}' option.  This option is ignored and should be removed from '#{configfile}'")
+                else
+                  raise("Unknown config parameter '#{key}'")
+                end
+              rescue ArgumentError => e
+                raise "Could not parse value for configuration option '#{key}' with value '#{val}'"
+              end
+            end
+          end
+        end
+
+        read_plugin_config_dir("#{@configdir}/plugin.d")
+
+        raise 'Identities can only match /\w\.\-/' unless @identity.match(/^[\w\.\-]+$/)
+
+        @configured = true
+
+        libdirs.each do |dir|
+          unless File.directory?(dir)
+            Log.debug("Cannot find libdir: #{dir}")
+          end
+
+          # remove the old one if it exists, we're moving it to the front
+          $LOAD_PATH.reject! { |elem| elem == dir }
+          $LOAD_PATH.unshift dir
+        end
+
+        if @logger_type == "syslog"
+          raise "The sylog logger is not usable on the Windows platform" if Util.windows?
+        end
+
+        PluginManager.loadclass("Mcollective::Facts::#{@factsource}_facts")
+        PluginManager.loadclass("Mcollective::Connector::#{@connector}")
+        PluginManager.loadclass("Mcollective::Security::#{@securityprovider}")
+        PluginManager << {:type => "global_stats", :class => RunnerStats.new}
+
+        Log.info("The Marionette Collective version #{MCollective::VERSION} started by #{$0} using config file #{configfile}")
+      else
+        raise("Cannot find config file '#{configfile}'")
+      end
+    end
+
+    def set_config_defaults(configfile)
+      @stomp = Hash.new
+      @subscribe = Array.new
+      @pluginconf = Hash.new
+      @connector = "base"
+      @securityprovider = "Base"
+      @factsource = "Yaml"
+      @identity = Socket.gethostname
+      @registration = "Base"
+      @registerinterval = 0
+      @registration_collective = nil
+      @registration_splay = false
+      @classesfile = "/var/lib/puppet/state/classes.txt"
+      @rpcaudit = false
+      @rpcauditprovider = ""
+      @rpcauthorization = false
+      @rpcauthprovider = ""
+      @configdir = File.dirname(configfile)
+      @color = !Util.windows?
+      @configfile = configfile
+      @logger_type = "file"
+      @keeplogs = 5
+      @max_log_size = 2097152
+      @rpclimitmethod = :first
+      @fact_cache_time = 300
+      @loglevel = "info"
+      @logfacility = "user"
+      @collectives = ["mcollective"]
+      @main_collective = @collectives.first
+      @ssl_cipher = "aes-256-cbc"
+      @direct_addressing = true
+      @direct_addressing_threshold = 10
+      @default_discovery_method = "mc"
+      @default_discovery_options = []
+      @ttl = 60
+      @mode = :client
+      @publish_timeout = 2
+      @threaded = false
+      @soft_shutdown = false
+      @soft_shutdown_timeout = nil
+      @activate_agents = true
+      @connection_timeout = nil
+      @default_batch_size = 0
+      @default_batch_sleep_time = 1
+    end
+
+    def libdir
+      $LOAD_PATH
+    end
+
+    def read_plugin_config_dir(dir)
+      return unless File.directory?(dir)
+
+      Dir.new(dir).each do |pluginconfigfile|
+        next unless pluginconfigfile =~ /^([\w]+).cfg$/
+
+        plugin = $1
+        File.open("#{dir}/#{pluginconfigfile}", "r").each do |line|
+          # strip blank lines
+          line.gsub!(/\s*$/, "")
+          next if line =~ /^#|^$/
+          if (line =~ /(.+?)\s*=\s*(.+)/)
+            key = $1.strip
+            val = $2
+            @pluginconf["#{plugin}.#{key}"] = val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/connector.rb

@@ -0,0 +1,18 @@
+module MCollective
+  # Connectors take care of transporting messages between clients and agents,
+  # the design doesn't your middleware to be very rich in features.  All it
+  # really needs is the ability to send and receive messages to named queues/topics.
+  #
+  # At present there are assumptions about the naming of topics and queues that is
+  # compatible with Stomp, ie.
+  #
+  # /topic/foo.bar/baz
+  # /queue/foo.bar/baz
+  #
+  # This is the only naming format that is supported, but you could replace Stomp
+  # with something else that supports the above, see MCollective::Connector::Stomp
+  # for the default connector.
+  module Connector
+    require "mcollective/connector/base"
+  end
+end

data/lib/mcollective/connector/base.rb

@@ -0,0 +1,26 @@
+module MCollective
+  # Connector plugins handle the communications with the middleware, you can provide your own to speak
+  # to something other than Stomp, your plugins must inherit from MCollective::Connector::Base and should
+  # provide the following methods:
+  #
+  # connect       - Creates a connection to the middleware, no arguments should get its parameters from the config
+  # receive       - Receive data from the middleware, should act like a blocking call only returning if/when data
+  #                 was received.  It should get data from all subscribed channels/topics.  Individual messages
+  #                 should be returned as MCollective::Request objects with the payload provided
+  # publish       - Takes a target and msg, should send the message to the supplied target topic or destination
+  # subscribe     - Adds a subscription to a specific message source
+  # unsubscribe   - Removes a subscription to a specific message source
+  # disconnect    - Disconnects from the middleware
+  #
+  # These methods are all that's needed for a new connector protocol and should hopefully be simple
+  # enough to not have tied us to Stomp.
+  module Connector
+    class Base
+      def self.inherited(klass)
+        plugin_name = klass.to_s.split("::").last.downcase
+        ddl = DDL.new(plugin_name, :connector)
+        PluginManager << {:type => "connector_plugin", :class => klass.to_s}
+      end
+    end
+  end
+end

data/lib/mcollective/data.rb

@@ -0,0 +1,91 @@
+module MCollective
+  module Data
+    require "mcollective/data/base"
+    require "mcollective/data/result"
+
+    def self.load_data_sources
+      PluginManager.find_and_load("data")
+
+      PluginManager.grep(/_data$/).each do |plugin|
+        begin
+          unless PluginManager[plugin].class.activate?
+            Log.debug("Disabling data plugin %s due to plugin activation policy" % plugin)
+            PluginManager.delete(plugin)
+          end
+        rescue Exception => e
+          Log.debug("Disabling data plugin %s due to exception #{e.class}: #{e}" % plugin)
+          PluginManager.delete(plugin)
+        end
+      end
+    end
+
+    def self.pluginname(plugin)
+      plugin.to_s =~ /_data$/i ? plugin.to_s.downcase : "%s_data" % plugin.to_s.downcase
+    end
+
+    def self.[](plugin)
+      PluginManager[pluginname(plugin)]
+    end
+
+    # Data.package("httpd").architecture
+    def self.method_missing(method, *args)
+      super unless PluginManager.include?(pluginname(method))
+
+      PluginManager[pluginname(method)].lookup(args.first)
+    end
+
+    def self.ddl(plugin)
+      DDL.new(pluginname(plugin), :data)
+    end
+
+    def self.ddl_validate(ddl, argument)
+      name = ddl.meta[:name]
+      query = ddl.entities[:data]
+
+      raise DDLValidationError, "No dataquery has been defined in the DDL for data plugin #{name}" unless query
+
+      input = query.fetch(:input, {})
+      output = query.fetch(:output, {})
+
+      raise DDLValidationError, "No output has been defined in the DDL for data plugin #{name}" if output.keys.empty?
+
+      if input[:query]
+        return true if argument.nil? && input[:query][:optional]
+
+        ddl.validate_input_argument(input, :query, argument)
+      else
+        raise("No data plugin argument was declared in the %s DDL but an input was supplied" % name) if argument
+        return true
+      end
+    end
+
+    def self.ddl_has_output?(ddl, output)
+      ddl.entities[:data][:output].include?(output.to_sym) rescue false
+    end
+
+    # For an input where the DDL requests a boolean or some number
+    # this will convert the input to the right type where possible
+    # else just returns the origin input unedited
+    #
+    # if anything here goes wrong just return the input value
+    # this is not really the end of the world or anything since
+    # all that will happen is that DDL validation will fail and
+    # the user will get an error, no need to be too defensive here
+    def self.ddl_transform_input(ddl, input)
+      begin
+        type = ddl.entities[:data][:input][:query][:type]
+
+        case type
+          when :boolean
+            return DDL.string_to_boolean(input)
+
+          when :number, :integer, :float
+            return DDL.string_to_number(input)
+        end
+      rescue
+      end
+
+      return input
+    end
+  end
+end

data/lib/mcollective/data/agent_data.ddl

@@ -0,0 +1,22 @@
+metadata    :name        => "Agent",
+            :description => "Meta data about installed MColletive Agents",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "ASL 2.0",
+            :version     => "1.0",
+            :url         => "https://docs.puppetlabs.com/mcollective/",
+            :timeout     => 1
+
+dataquery :description => "Agent Meta Data" do
+    input :query,
+          :prompt => "Agent Name",
+          :description => "Valid agent name",
+          :type => :string,
+          :validation => /^[\w\_]+$/,
+          :maxlength => 20
+
+    [:license, :timeout, :description, :url, :version, :author].each do |item|
+      output item,
+             :description => "Agent #{item}",
+             :display_as => item.to_s.capitalize
+    end
+end

data/lib/mcollective/data/agent_data.rb

@@ -0,0 +1,17 @@
+module MCollective
+  module Data
+    class Agent_data<Base
+      query do |plugin|
+        raise "No agent called #{plugin} found" unless PluginManager.include?("#{plugin}_agent")
+
+        agent = PluginManager["#{plugin}_agent"]
+
+        result[:agent] = plugin
+
+        [:license, :timeout, :description, :url, :version, :author].each do |item|
+          result[item] = agent.meta[item]
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/data/base.rb

@@ -0,0 +1,67 @@
+module MCollective
+  module Data
+    class Base
+      attr_reader :name, :result, :ddl, :timeout
+
+      # Register plugins that inherits base
+      def self.inherited(klass)
+        type = klass.to_s.split("::").last.downcase
+
+        PluginManager << {:type => type, :class => klass.to_s, :single_instance => false}
+      end
+
+      def initialize
+        @name = self.class.to_s.split("::").last.downcase
+        @ddl = DDL.new(@name, :data)
+        @result = Result.new(@ddl.dataquery_interface[:output])
+        @timeout = @ddl.meta[:timeout] || 1
+
+        startup_hook
+      end
+
+      def lookup(what)
+        ddl_validate(what)
+
+        Log.debug("Doing data query %s for '%s'" % [ @name, what ])
+
+        Timeout::timeout(@timeout) do
+          query_data(what)
+        end
+
+        @result
+      rescue Timeout::Error
+        # Timeout::Error is a inherited from Interrupt which seems a really
+        # strange choice, making it an equivelant of ^C and such.  Catch it
+        # and raise something less critical that will not the runner to just
+        # give up the ghost
+        msg = "Data plugin %s timed out on query '%s'" % [@name, what]
+        Log.error(msg)
+        raise MsgTTLExpired, msg
+      end
+
+      def self.query(&block)
+        self.module_eval { define_method("query_data", &block) }
+      end
+
+      def ddl_validate(what)
+        Data.ddl_validate(@ddl, what)
+      end
+
+      # activate_when do
+      #    file.exist?("/usr/bin/puppet")
+      # end
+      def self.activate_when(&block)
+        (class << self; self; end).instance_eval do
+          define_method("activate?", &block)
+        end
+      end
+
+      # Always be active unless a specific block is given with activate_when
+      def self.activate?
+        return true
+      end
+
+      def startup_hook;end
+    end
+  end
+end