choria-mcorpc-support 0.0.1

data/lib/mcollective/message.rb

@@ -0,0 +1,248 @@
+module MCollective
+  # container for a message, its headers, agent, collective and other meta data
+  class Message
+    attr_reader :message, :request, :validated, :msgtime, :payload, :type, :expected_msgid, :reply_to
+    attr_accessor :headers, :agent, :collective, :filter
+    attr_accessor :requestid, :discovered_hosts, :options, :ttl
+
+    VALIDTYPES = [:message, :request, :direct_request, :reply]
+
+    # payload                  - the message body without headers etc, just the text
+    # message                  - the original message received from the middleware
+    # options[:base64]         - if the body base64 encoded?
+    # options[:agent]          - the agent the message is for/from
+    # options[:collective]     - the collective its for/from
+    # options[:headers]        - the message headers
+    # options[:type]           - an indicator about the type of message, :message, :request, :direct_request or :reply
+    # options[:request]        - if this is a reply this should old the message we are replying to
+    # options[:filter]         - for requests, the filter to encode into the message
+    # options[:options]        - the normal client options hash
+    # options[:ttl]            - the maximum amount of seconds this message can be valid for
+    # options[:expected_msgid] - in the case of replies this is the msgid it is expecting in the replies
+    # options[:requestid]      - specific request id to use else one will be generated
+    def initialize(payload, message, options = {})
+      options = {:base64 => false,
+                 :agent => nil,
+                 :headers => {},
+                 :type => :message,
+                 :request => nil,
+                 :filter => Util.empty_filter,
+                 :options => {},
+                 :ttl => 60,
+                 :expected_msgid => nil,
+                 :requestid => nil,
+                 :collective => nil}.merge(options)
+
+      @payload = payload
+      @message = message
+      @requestid = options[:requestid]
+      @discovered_hosts = nil
+      @reply_to = nil
+
+      @type = options[:type]
+      @headers = options[:headers]
+      @base64 = options[:base64]
+      @filter = options[:filter]
+      @expected_msgid = options[:expected_msgid]
+      @options = options[:options]
+
+      @ttl = @options[:ttl] || Config.instance.ttl
+      @msgtime = 0
+
+      @validated = false
+
+      if options[:request]
+        @request = options[:request]
+        @agent = request.agent
+        @collective = request.collective
+        @type = :reply
+      else
+        @agent = options[:agent]
+        @collective = options[:collective]
+      end
+
+      base64_decode!
+    end
+
+    # Sets the message type to one of the known types.  In the case of :direct_request
+    # the list of hosts to communicate with should have been set with #discovered_hosts
+    # else an exception will be raised.  This is for extra security, we never accidentally
+    # want to send a direct request without a list of hosts or something weird like that
+    # as it might result in a filterless broadcast being sent.
+    #
+    # Additionally you simply cannot set :direct_request if direct_addressing was not enabled
+    # this is to force a workflow that doesnt not yield in a mistake when someone might assume
+    # direct_addressing is enabled when its not.
+    def type=(type)
+      raise "Unknown message type #{type}" unless VALIDTYPES.include?(type)
+
+      if type == :direct_request
+        raise "Direct requests is not enabled using the direct_addressing config option" unless Config.instance.direct_addressing
+
+        unless @discovered_hosts && !@discovered_hosts.empty?
+          raise "Can only set type to :direct_request if discovered_hosts have been set"
+        end
+
+        # clear out the filter, custom discovery sources might interpret the filters
+        # different than the remote mcollectived and in directed mode really the only
+        # filter that matters is the agent filter
+        @filter = Util.empty_filter
+        @filter["agent"] << @agent
+      end
+
+      @type = type
+    end
+
+    # Sets a custom reply-to target for requests.  The connector plugin should inspect this
+    # when constructing requests and set this header ensuring replies will go to the custom target
+    # otherwise the connector should just do what it usually does
+    def reply_to=(target)
+      raise "Custom reply targets can only be set on requests" unless [:request, :direct_request].include?(@type)
+
+      @reply_to = target
+    end
+
+    # in the case of reply messages we are expecting replies to a previously
+    # created message.  This stores a hint to that previously sent message id
+    # and can be used by other classes like the security plugins as a means
+    # of optimizing their behavior like by ignoring messages not directed
+    # at us.
+    def expected_msgid=(msgid)
+      raise "Can only store the expected msgid for reply messages" unless @type == :reply
+      @expected_msgid = msgid
+    end
+
+    def base64_decode!
+      return unless @base64
+
+      @payload = SSL.base64_decode(@payload)
+      @base64 = false
+    end
+
+    def base64_encode!
+      return if @base64
+
+      @payload = SSL.base64_encode(@payload)
+      @base64 = true
+    end
+
+    def base64?
+      @base64
+    end
+
+    def description
+      cid = ""
+      cid += payload[:callerid] + "@" if payload.include?(:callerid)
+      cid += payload[:senderid]
+
+      "#{requestid} for agent '#{agent}' in collective '#{collective}' from #{cid}"
+    end
+
+    def encode!
+      case type
+        when :reply
+          raise "Cannot encode a reply message if no request has been associated with it" unless request
+          raise 'callerid in original request is not valid, surpressing reply to potentially forged request' unless PluginManager["security_plugin"].valid_callerid?(request.payload[:callerid])
+
+          @requestid = request.payload[:requestid]
+          @payload = PluginManager["security_plugin"].encodereply(agent, payload, requestid, request.payload[:callerid])
+        when :request, :direct_request
+          validate_compound_filter(@filter["compound"]) unless @filter["compound"].empty?
+
+          @requestid = create_reqid unless @requestid
+          @payload = PluginManager["security_plugin"].encoderequest(Config.instance.identity, payload, requestid, filter, agent, collective, ttl)
+        else
+          raise "Cannot encode #{type} messages"
+      end
+    end
+
+    def validate_compound_filter(compound_filter)
+      compound_filter.each do |filter|
+        filter.each do |statement|
+          if statement["fstatement"]
+            functionname = statement["fstatement"]["name"]
+            pluginname = Data.pluginname(functionname)
+            value = statement["fstatement"]["value"]
+
+            ddl = DDL.new(pluginname, :data)
+
+            # parses numbers and booleans entered as strings into proper
+            # types of data so that DDL validation will pass
+            statement["fstatement"]["params"] = Data.ddl_transform_input(ddl, statement["fstatement"]["params"])
+
+            Data.ddl_validate(ddl, statement["fstatement"]["params"])
+
+            unless value && Data.ddl_has_output?(ddl, value)
+              DDL.validation_fail!(:PLMC41, "Data plugin '%{functionname}()' does not return a '%{value}' value", :error, {:functionname => functionname, :value => value})
+            end
+          end
+        end
+      end
+    end
+
+    def decode!
+      raise "Cannot decode message type #{type}" unless [:request, :reply].include?(type)
+
+      begin
+        @payload = PluginManager["security_plugin"].decodemsg(self)
+      rescue Exception => e
+        if type == :request
+          # If we're a server receiving a request, reraise
+          raise(e)
+        else
+          # We're in the client, log and carry on as best we can
+
+          # Note: mc_sender is unverified.  The verified identity is in the
+          # payload we just failed to decode
+          Log.warn("Failed to decode a message from '#{headers["mc_sender"]}': #{e}")
+          return
+        end
+      end
+
+      if type == :request
+        raise 'callerid in request is not valid, surpressing reply to potentially forged request' unless PluginManager["security_plugin"].valid_callerid?(payload[:callerid])
+      end
+
+      [:collective, :agent, :filter, :requestid, :ttl, :msgtime].each do |prop|
+        instance_variable_set("@#{prop}", payload[prop]) if payload.include?(prop)
+      end
+    end
+
+    # Perform validation against the message by checking filters and ttl
+    def validate
+      raise "Can only validate request messages" unless type == :request
+
+      msg_age = Time.now.utc.to_i - msgtime
+
+      if msg_age > ttl
+        PluginManager["global_stats"].ttlexpired
+        raise(MsgTTLExpired, "Message #{description} created at #{msgtime} is #{msg_age} seconds old, TTL is #{ttl}. Rejecting message.")
+      end
+
+      raise(NotTargettedAtUs, "Message #{description} does not pass filters. Ignoring message.") unless PluginManager["security_plugin"].validate_filter?(payload[:filter])
+
+      @validated = true
+    end
+
+    # publish a reply message by creating a target name and sending it
+    def publish
+      # If we've been specificaly told about hosts that were discovered
+      # use that information to do P2P calls if appropriate else just
+      # send it as is.
+      config = Config.instance
+      if @discovered_hosts && config.direct_addressing && (@discovered_hosts.size <= config.direct_addressing_threshold)
+        self.type = :direct_request
+        Log.debug("Handling #{requestid} as a direct request")
+      end
+
+      PluginManager['connector_plugin'].publish(self)
+    end
+
+    def create_reqid
+      # we gsub out the -s so that the format of the id does not
+      # change from previous versions, these should just be more
+      # unique than previous ones
+      SSL.uuid.gsub("-", "")
+    end
+  end
+end

data/lib/mcollective/monkey_patches.rb

@@ -0,0 +1,152 @@
+# start_with? was introduced in 1.8.7, we need to support
+# 1.8.5 and 1.8.6
+class String
+  def start_with?(str)
+    return self[0..(str.length-1)] == str
+  end unless method_defined?("start_with?")
+end
+
+# Make arrays of Symbols sortable
+class Symbol
+  include Comparable
+
+  def <=>(other)
+    self.to_s <=> other.to_s
+  end unless method_defined?("<=>")
+end
+
+# This provides an alias for RbConfig to Config for versions of Ruby older then
+# # version 1.8.5. This allows us to use RbConfig in place of the older Config in
+# # our code and still be compatible with at least Ruby 1.8.1.
+# require 'rbconfig'
+unless defined? ::RbConfig
+  ::RbConfig = ::Config
+end
+
+# a method # that walks an array in groups, pass a block to
+# call the block on each sub array
+class Array
+  def in_groups_of(chunk_size, padded_with=nil, &block)
+    arr = self.clone
+
+    # how many to add
+    padding = chunk_size - (arr.size % chunk_size)
+
+    # pad at the end
+    arr.concat([padded_with] * padding) unless padding == chunk_size
+
+    # how many chunks we'll make
+    count = arr.size / chunk_size
+
+    # make that many arrays
+    result = []
+    count.times {|s| result <<  arr[s * chunk_size, chunk_size]}
+
+    if block_given?
+      result.each_with_index do |a, i|
+        case block.arity
+          when 1
+            yield(a)
+          when 2
+            yield(a, (i == result.size - 1))
+          else
+            raise "Expected 1 or 2 arguments, got #{block.arity}"
+        end
+      end
+    else
+      result
+    end
+  end unless method_defined?(:in_groups_of)
+end
+
+class String
+  def bytes(&block)
+    # This should not be necessary, really ...
+    require 'enumerator'
+    return to_enum(:each_byte) unless block_given?
+    each_byte(&block)
+  end unless method_defined?(:bytes)
+end
+
+class Dir
+  def self.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+    case prefix_suffix
+    when nil
+      prefix = "d"
+      suffix = ""
+    when String
+      prefix = prefix_suffix
+      suffix = ""
+    when Array
+      prefix = prefix_suffix[0]
+      suffix = prefix_suffix[1]
+    else
+      raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+    end
+    tmpdir ||= Dir.tmpdir
+    t = Time.now.strftime("%Y%m%d")
+    n = nil
+    begin
+      path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+      path << "-#{n}" if n
+      path << suffix
+      Dir.mkdir(path, 0700)
+    rescue Errno::EEXIST
+      n ||= 0
+      n += 1
+      retry
+    end
+
+    if block_given?
+      begin
+        yield path
+      ensure
+        FileUtils.remove_entry_secure path
+      end
+    else
+      path
+    end
+  end unless method_defined?(:mktmpdir)
+
+  def self.tmpdir
+    tmp = '.'
+    for dir in [ENV['TMPDIR'], ENV['TMP'], ENV['TEMP'], '/tmp']
+      if dir and stat = File.stat(dir) and stat.directory? and stat.writable?
+        tmp = dir
+        break
+      end rescue nil
+    end
+    File.expand_path(tmp)
+  end unless method_defined?(:tmpdir)
+end
+
+# Reject all SSLv2 ciphers and all SSLv2 or SSLv3 handshakes by default
+require 'openssl'
+class OpenSSL::SSL::SSLContext
+  if DEFAULT_PARAMS[:options]
+    DEFAULT_PARAMS[:options] |= OpenSSL::SSL::OP_NO_SSLv2 | OpenSSL::SSL::OP_NO_SSLv3
+  else
+    DEFAULT_PARAMS[:options] = OpenSSL::SSL::OP_NO_SSLv2 | OpenSSL::SSL::OP_NO_SSLv3
+  end
+
+  # ruby 1.8.5 doesn't define this constant, but has it on by default
+  if defined?(OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS)
+    DEFAULT_PARAMS[:options] |= OpenSSL::SSL::OP_DONT_INSERT_EMPTY_FRAGMENTS
+  end
+
+  if DEFAULT_PARAMS[:ciphers]
+    DEFAULT_PARAMS[:ciphers] << ':!SSLv2'
+  end
+
+  alias __mcollective_original_initialize initialize
+  private :__mcollective_original_initialize
+
+  def initialize(*args)
+    __mcollective_original_initialize(*args)
+    params = {
+      :options => DEFAULT_PARAMS[:options],
+      :ciphers => DEFAULT_PARAMS[:ciphers],
+    }
+    set_params(params)
+  end
+end

data/lib/mcollective/optionparser.rb

@@ -0,0 +1,197 @@
+module MCollective
+  # A simple helper to build cli tools that supports a uniform command line
+  # layout.
+  class Optionparser
+    attr_reader :parser
+
+    # Creates a new instance of the parser, you can supply defaults and include named groups of options.
+    #
+    # Starts a parser that defaults to verbose and that includs the filter options:
+    #
+    #  oparser = MCollective::Optionparser.new({:verbose => true}, "filter")
+    #
+    # Stats a parser in non verbose mode that does support discovery
+    #
+    #  oparser = MCollective::Optionparser.new()
+    #
+    # Starts a parser in verbose mode that does not show the common options:
+    #
+    #  oparser = MCollective::Optionparser.new({:verbose => true}, "filter", "common")
+    def initialize(defaults = {}, include_sections = nil, exclude_sections = nil)
+      @parser = ::OptionParser.new
+
+      @include = [include_sections].flatten
+      @exclude = [exclude_sections].flatten
+
+      @options = Util.default_options
+
+      @options.merge!(defaults)
+    end
+
+    # Parse the options returning the options, you can pass a block that adds additional options
+    # to the Optionparser.
+    #
+    # The sample below starts a parser that also prompts for --arguments in addition to the defaults.
+    # It also sets the description and shows a usage message specific to this app.
+    #
+    #  options = oparser.parse{|parser, options|
+    #       parser.define_head "Control the mcollective controller daemon"
+    #       parser.banner = "Usage: sh-mcollective [options] command"
+    #
+    #       parser.on('--arg', '--argument ARGUMENT', 'Argument to pass to agent') do |v|
+    #           options[:argument] = v
+    #       end
+    #  }
+    #
+    # Users can set default options that get parsed in using the MCOLLECTIVE_EXTRA_OPTS environemnt
+    # variable
+    def parse(&block)
+      yield(@parser, @options) if block_given?
+
+      add_required_options
+
+      add_common_options unless @exclude.include?("common")
+
+      @include.each do |i|
+        next if @exclude.include?(i)
+
+        options_name = "add_#{i}_options"
+        send(options_name)  if respond_to?(options_name)
+      end
+
+      @parser.environment("MCOLLECTIVE_EXTRA_OPTS")
+
+      @parser.parse!
+
+      @options[:collective] = Config.instance.main_collective unless @options[:collective]
+
+      @options
+    end
+
+    # These options will be added if you pass 'filter' into the include list of the
+    # constructor.
+    def add_filter_options
+      @parser.separator ""
+      @parser.separator "Host Filters"
+
+      @parser.on('-W', '--with FILTER', 'Combined classes and facts filter') do |f|
+        f.split(" ").each do |filter|
+          begin
+            fact_parsed = parse_fact(filter)
+            @options[:filter]["fact"] << fact_parsed
+          rescue
+            @options[:filter]["cf_class"] << filter
+          end
+        end
+      end
+
+      @parser.on('-S', '--select FILTER', 'Compound filter combining facts and classes') do |f|
+        @options[:filter]["compound"] << Matcher.create_compound_callstack(f)
+      end
+
+      @parser.on('-F', '--wf', '--with-fact fact=val', 'Match hosts with a certain fact') do |f|
+        fact_parsed = parse_fact(f)
+
+        @options[:filter]["fact"] << fact_parsed if fact_parsed
+      end
+
+      @parser.on('-C', '--wc', '--with-class CLASS', 'Match hosts with a certain config management class') do |f|
+        @options[:filter]["cf_class"] << f
+      end
+
+      @parser.on('-A', '--wa', '--with-agent AGENT', 'Match hosts with a certain agent') do |a|
+        @options[:filter]["agent"] << a
+      end
+
+      @parser.on('-I', '--wi', '--with-identity IDENT', 'Match hosts with a certain configured identity') do |a|
+        @options[:filter]["identity"] << a
+      end
+    end
+
+    # These options should always be present
+    def add_required_options
+      @parser.on('-c', '--config FILE', 'Load configuration from file rather than default') do |f|
+        @options[:config] = f
+      end
+
+      @parser.on('-v', '--verbose', 'Be verbose') do |v|
+        @options[:verbose] = v
+      end
+
+      @parser.on('-h', '--help', 'Display this screen') do
+        puts @parser
+        exit! 1
+      end
+    end
+
+    # These options will be added to most cli tools
+    def add_common_options
+      @parser.separator ""
+      @parser.separator "Common Options"
+
+      @parser.on('-T', '--target COLLECTIVE', 'Target messages to a specific sub collective') do |f|
+        @options[:collective] = f
+      end
+
+      @parser.on('--dt', '--discovery-timeout SECONDS', Integer, 'Timeout for doing discovery') do |t|
+        @options[:disctimeout] = t
+      end
+
+      @parser.on('-t', '--timeout SECONDS', Integer, 'Timeout for calling remote agents') do |t|
+        @options[:timeout] = t
+      end
+
+      @parser.on('-q', '--quiet', 'Do not be verbose') do |v|
+        @options[:verbose] = false
+      end
+
+      @parser.on('--ttl TTL', 'Set the message validity period') do |v|
+        @options[:ttl] = v.to_i
+      end
+
+      @parser.on('--reply-to TARGET', 'Set a custom target for replies') do |v|
+        @options[:reply_to] = v
+      end
+
+      @parser.on('--dm', '--disc-method METHOD', 'Which discovery method to use') do |v|
+        raise "Discovery method is already set by a competing option" if @options[:discovery_method] && @options[:discovery_method] != v
+        @options[:discovery_method] = v
+      end
+
+      @parser.on('--do', '--disc-option OPTION', 'Options to pass to the discovery method') do |a|
+        @options[:discovery_options] << a
+      end
+
+      @parser.on("--nodes FILE", "List of nodes to address") do |v|
+        raise "Cannot mix --disc-method, --disc-option and --nodes" if @options[:discovery_method] || @options[:discovery_options].size > 0
+        raise "Cannot read the discovery file #{v}" unless File.readable?(v)
+
+        @options[:discovery_method] = "flatfile"
+        @options[:discovery_options] << v
+      end
+
+      @parser.on("--publish_timeout TIMEOUT", Integer, "Timeout for publishing requests to remote agents.") do |pt|
+        @options[:publish_timeout] = pt
+      end
+
+      @parser.on("--threaded", "Start publishing requests and receiving responses in threaded mode.") do |v|
+        @options[:threaded] = true
+      end
+
+      @parser.on("--sort", "Sort the output of an RPC call before processing.") do |v|
+        @options[:sort] = true
+      end
+
+      @parser.on("--connection-timeout TIMEOUT", Integer, "Set the timeout for establishing a connection to the middleware") do |v|
+        @options[:connection_timeout] = Integer(v)
+      end
+    end
+
+    private
+    # Parse a fact filter string like foo=bar into the tuple hash thats needed
+    def parse_fact(fact)
+      Util.parse_fact_string(fact)
+    end
+
+  end
+end