mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/configuration/dotted_hash.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Configuration
+      class DottedHash < ::Hash
+        def initialize(hash, keep_nesting=false)
+          # Add the hash keys to our collection explicitly so they survive the
+          # dot flattening.  This is typical for full config source instances,
+          # but not for uses of DottedHash serializing for transmission.
+          self.merge!(hash) if keep_nesting
+
+          self.merge!(dot_flattened(hash))
+          DottedHash.symbolize(self)
+        end
+
+        def inspect
+          "#<#{self.class.name}:#{object_id} #{super}>"
+        end
+
+        def to_hash
+          {}.replace(self)
+        end
+
+        def self.symbolize(hash)
+          hash.keys.each do |key|
+            hash[key.to_sym] = hash.delete(key)
+          end
+        end
+
+        protected
+        # turns {'a' => {'b' => 'c'}} into {'a.b' => 'c'}
+        def dot_flattened(nested_hash, names=[], result={})
+          nested_hash.each do |key, val|
+            next if val == nil
+            if val.respond_to?(:has_key?)
+              dot_flattened(val, names + [key], result)
+            else
+              result[(names + [key]).join('.')] = val
+            end
+          end
+          result
+        end
+      end
+
+    end
+  end
+end

data/lib/mmtrix/agent/configuration/environment_source.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Configuration
+      class EnvironmentSource < DottedHash
+        SUPPORTED_PREFIXES = /^mmtrix_|^mmtrix_/i
+        SPECIAL_CASE_KEYS  = [
+          'MM_TRIX_ENV', # read by Mmtrix::Control::Frameworks::Ruby
+          'MM_TRIX_LOG'  # read by set_log_file
+        ]
+
+        attr_accessor :alias_map, :type_map
+
+        def initialize
+          set_log_file
+          set_config_file
+
+          @alias_map = {}
+          @type_map = {}
+
+          DEFAULTS.each do |config_setting, value|
+            self.type_map[config_setting] = value[:type]
+            set_aliases(config_setting, value)
+          end
+
+          set_values_from_mmtrix_environment_variables
+        end
+
+        def set_aliases(config_setting, value)
+          set_dotted_alias(config_setting)
+
+          return unless value[:aliases]
+          value[:aliases].each do |config_alias|
+            self.alias_map[config_alias] = config_setting
+          end
+        end
+
+        def set_dotted_alias(original_config_setting)
+          config_setting = original_config_setting.to_s
+
+          if config_setting.include? '.'
+            config_alias = config_setting.gsub(/\./,'_').to_sym
+            self.alias_map[config_alias] = original_config_setting
+          end
+        end
+
+        def set_log_file
+          if ENV['MM_TRIX_LOG']
+            if ENV['MM_TRIX_LOG'].upcase == 'STDOUT'
+              self[:log_file_path] = self[:log_file_name] = 'STDOUT'
+            else
+              self[:log_file_path] = File.dirname(ENV['MM_TRIX_LOG'])
+              self[:log_file_name] = File.basename(ENV['MM_TRIX_LOG'])
+            end
+          end
+        end
+
+        def set_config_file
+          self[:config_path] = ENV['NRCONFIG'] if ENV['NRCONFIG']
+        end
+
+        def set_values_from_mmtrix_environment_variables
+          nr_env_var_keys = collect_mmtrix_environment_variable_keys
+
+          nr_env_var_keys.each do |key|
+            next if SPECIAL_CASE_KEYS.include?(key.upcase)
+            set_value_from_environment_variable(key)
+          end
+        end
+
+        def set_value_from_environment_variable(key)
+          config_key = convert_environment_key_to_config_key(key)
+          set_key_by_type(config_key, key)
+        end
+
+        def set_key_by_type(config_key, environment_key)
+          value = ENV[environment_key]
+          type = self.type_map[config_key]
+
+          if type == String
+            self[config_key] = value
+          elsif type == Fixnum
+            self[config_key] = value.to_i
+          elsif type == Float
+            self[config_key] = value.to_f
+          elsif type == Symbol
+            self[config_key] = value.to_sym
+          elsif type == Mmtrix::Agent::Configuration::Boolean
+            if value =~ /false|off|no/i
+              self[config_key] = false
+            elsif value != nil
+              self[config_key] = true
+            end
+          else
+            ::Mmtrix::Agent.logger.info("#{environment_key} does not have a corresponding configuration setting (#{config_key} does not exist).")
+            ::Mmtrix::Agent.logger.info("Run `rake mmtrix:config:docs` or visit https://mmtrix.com/docs/ruby/ruby-agent-configuration to see a list of available configuration settings.")
+            self[config_key] = value
+          end
+        end
+
+        def convert_environment_key_to_config_key(key)
+          stripped_key = key.gsub(SUPPORTED_PREFIXES, '').downcase.to_sym
+          self.alias_map[stripped_key] || stripped_key
+        end
+
+        def collect_mmtrix_environment_variable_keys
+          ENV.keys.select { |key| key.match(SUPPORTED_PREFIXES) }
+        end
+
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/configuration/high_security_source.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/configuration/dotted_hash'
+
+module Mmtrix
+  module Agent
+    module Configuration
+      class HighSecuritySource < DottedHash
+        def initialize(local_settings)
+          super({
+            :ssl => true,
+
+            :capture_params           => false,
+            :'resque.capture_params'  => false,
+            :'sidekiq.capture_params' => false,
+
+            :'transaction_tracer.record_sql' => record_sql_setting(local_settings, :'transaction_tracer.record_sql'),
+            :'slow_sql.record_sql'           => record_sql_setting(local_settings, :'slow_sql.record_sql'),
+            :'mongo.obfuscate_queries'       => true,
+
+            :'custom_insights_events.enabled'   => false,
+            :'strip_exception_messages.enabled' => true
+          })
+        end
+
+        OFF = "off".freeze
+        RAW = "raw".freeze
+        OBFUSCATED = "obfuscated".freeze
+
+        SET_TO_OBFUSCATED = [RAW, OBFUSCATED]
+
+        def record_sql_setting(local_settings, key)
+          original_value  = local_settings[key]
+          result = if SET_TO_OBFUSCATED.include?(original_value)
+            OBFUSCATED
+          else
+            OFF
+          end
+
+          if result != original_value
+            Mmtrix::Agent.logger.info("Disabling setting #{key}='#{original_value}' because high security mode is enabled. Value will be '#{result}'")
+          end
+
+          result
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/configuration/manager.rb

@@ -0,0 +1,395 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'forwardable'
+require 'mmtrix/agent/configuration/mask_defaults'
+require 'mmtrix/agent/configuration/yaml_source'
+require 'mmtrix/agent/configuration/default_source'
+require 'mmtrix/agent/configuration/server_source'
+require 'mmtrix/agent/configuration/environment_source'
+require 'mmtrix/agent/configuration/high_security_source'
+
+module Mmtrix
+  module Agent
+    module Configuration
+      class Manager
+
+        # Defining these explicitly saves object allocations that we incur
+        # if we use Forwardable and def_delegators.
+        def [](key)
+          @cache[key]
+        end
+
+        def has_key?(key)
+          @cache.has_key?[key]
+        end
+
+        def keys
+          @cache.keys
+        end
+
+        def initialize
+          reset_to_defaults
+          @callbacks = Hash.new {|hash,key| hash[key] = [] }
+        end
+
+        def add_config_for_testing(source, level=0)
+          raise 'Invalid config type for testing' unless [Hash, DottedHash].include?(source.class)
+          invoke_callbacks(:add, source)
+          @configs_for_testing << [source.freeze, level]
+          reset_cache
+          log_config(:add, source)
+        end
+
+        def remove_config_type(sym)
+          source = case sym
+          when :high_security then @high_security_source
+          when :environment   then @environment_source
+          when :server        then @server_source
+          when :manual        then @manual_source
+          when :yaml          then @yaml_source
+          when :default       then @default_source
+          end
+
+          remove_config(source)
+        end
+
+        def remove_config(source)
+          case source
+          when HighSecuritySource then @high_security_source = nil
+          when EnvironmentSource  then @environment_source   = nil
+          when ServerSource       then @server_source        = nil
+          when ManualSource       then @manual_source        = nil
+          when YamlSource         then @yaml_source          = nil
+          when DefaultSource      then @default_source       = nil
+          else
+            @configs_for_testing.delete_if {|src,lvl| src == source}
+          end
+
+          reset_cache
+          invoke_callbacks(:remove, source)
+          log_config(:remove, source)
+        end
+
+        def replace_or_add_config(source)
+          source.freeze
+          was_finished = finished_configuring?
+
+          invoke_callbacks(:add, source)
+          case source
+          when HighSecuritySource then @high_security_source = source
+          when EnvironmentSource  then @environment_source   = source
+          when ServerSource       then @server_source        = source
+          when ManualSource       then @manual_source        = source
+          when YamlSource         then @yaml_source          = source
+          when DefaultSource      then @default_source       = source
+          else
+            Mmtrix::Agent.logger.warn("Invalid config format; config will be ignored: #{source}")
+          end
+
+          reset_cache
+          log_config(:add, source)
+
+          notify_finished_configuring if !was_finished && finished_configuring?
+        end
+
+        def source(key)
+          config_stack.each do |config|
+            if config.respond_to?(key.to_sym) || config.has_key?(key.to_sym)
+              return config
+            end
+          end
+        end
+
+        def fetch(key)
+          config_stack.each do |config|
+            next unless config
+            accessor = key.to_sym
+
+            if config.has_key?(accessor)
+              evaluated = evaluate_procs(config[accessor])
+
+              begin
+                return apply_transformations(accessor, evaluated)
+              rescue
+                next
+              end
+            end
+          end
+
+          nil
+        end
+
+        def evaluate_procs(value)
+          if value.respond_to?(:call)
+            instance_eval(&value)
+          else
+            value
+          end
+        end
+
+        def apply_transformations(key, value)
+          if transform = transform_from_default(key)
+            begin
+              transform.call(value)
+            rescue => e
+              ::Mmtrix::Agent.logger.error("Error applying transformation for #{key}, pre-transform value was: #{value}.", e)
+              raise e
+            end
+          else
+            value
+          end
+        end
+
+        def transform_from_default(key)
+          ::Mmtrix::Agent::Configuration::DefaultSource.transform_for(key)
+        end
+
+        def register_callback(key, &proc)
+          @callbacks[key] << proc
+          proc.call(@cache[key])
+        end
+
+        def invoke_callbacks(direction, source)
+          return unless source
+          source.keys.each do |key|
+
+            if @cache[key] != source[key]
+              @callbacks[key].each do |proc|
+                if direction == :add
+                  proc.call(source[key])
+                else
+                  proc.call(@cache[key])
+                end
+              end
+            end
+          end
+        end
+
+        def notify_finished_configuring
+          Mmtrix::Agent.instance.events.notify(:finished_configuring)
+        end
+
+        def finished_configuring?
+          !@server_source.nil?
+        end
+
+        def flattened
+          config_stack.reverse.inject({}) do |flat,layer|
+            thawed_layer = layer.to_hash.dup
+            thawed_layer.each do |k,v|
+              begin
+                thawed_layer[k] = instance_eval(&v) if v.respond_to?(:call)
+              rescue => e
+                ::Mmtrix::Agent.logger.debug("#{e.class.name} : #{e.message} - when accessing config key #{k}")
+                thawed_layer[k] = nil
+              end
+              thawed_layer.delete(:config)
+            end
+            flat.merge(thawed_layer.to_hash)
+          end
+        end
+
+        def apply_mask(hash)
+          MASK_DEFAULTS. \
+            select {|_, proc| proc.call}. \
+            each {|key, _| hash.delete(key) }
+          hash
+        end
+
+        def to_collector_hash
+          DottedHash.new(apply_mask(flattened)).to_hash.delete_if do |k, v|
+            default = DEFAULTS[k]
+            if default
+              default[:exclude_from_reported_settings]
+            else
+              # In our tests, we add totally bogus configs, because testing.
+              # In those cases, there will be no default. So we'll just let
+              # them through.
+              false
+            end
+          end
+        end
+
+        def app_names
+          case Mmtrix::Agent.config[:app_name]
+          when Array then Mmtrix::Agent.config[:app_name]
+          when String then Mmtrix::Agent.config[:app_name].split(';')
+          else []
+          end
+        end
+
+        MALFORMED_LABELS_WARNING = "Skipping malformed labels configuration"
+        PARSING_LABELS_FAILURE   = "Failure during parsing labels. Ignoring and carrying on with connect."
+
+        MAX_LABEL_COUNT  = 64
+        MAX_LABEL_LENGTH = 255
+
+        def parsed_labels
+          case Mmtrix::Agent.config[:labels]
+          when String
+            parse_labels_from_string
+          else
+            parse_labels_from_dictionary
+          end
+        rescue => e
+          Mmtrix::Agent.logger.error(PARSING_LABELS_FAILURE, e)
+          []
+        end
+
+        def parse_labels_from_string
+          labels = Mmtrix::Agent.config[:labels]
+          label_pairs = break_label_string_into_pairs(labels)
+          make_label_hash(label_pairs, labels)
+        end
+
+        def break_label_string_into_pairs(labels)
+          stripped_labels = labels.strip
+          stripped_labels.split(';').map do |pair|
+            pair.split(':').map(&:strip)
+          end
+        end
+
+        def valid_label_pairs?(label_pairs)
+          label_pairs.all? do |pair|
+            pair.length == 2 &&
+              valid_label_item?(pair.first) &&
+              valid_label_item?(pair.last)
+          end
+        end
+
+        def valid_label_item?(item)
+          case item
+          when String  then !item.empty?
+          when Numeric then true
+          when true    then true
+          when false   then true
+          else false
+          end
+        end
+
+        def make_label_hash(pairs, labels = nil)
+          # This can accept a hash, so force it down to an array of pairs first
+          pairs = Array(pairs)
+
+          unless valid_label_pairs?(pairs)
+            Mmtrix::Agent.logger.warn("#{MALFORMED_LABELS_WARNING}: #{labels||pairs}")
+            return []
+          end
+
+          pairs = limit_number_of_labels(pairs)
+          pairs = remove_duplicates(pairs)
+          pairs.map do |key, value|
+            {
+              'label_type'  => truncate(key),
+              'label_value' => truncate(value.to_s, key)
+            }
+          end
+        end
+
+        def truncate(text, key=nil)
+          if text.length > MAX_LABEL_LENGTH
+            if key
+              msg = "The value for the label '#{key}' is longer than the allowed #{MAX_LABEL_LENGTH} and will be truncated. Value = '#{text}'"
+            else
+              msg = "Label name longer than the allowed #{MAX_LABEL_LENGTH} will be truncated. Name = '#{text}'"
+            end
+            Mmtrix::Agent.logger.warn(msg)
+            text[0..MAX_LABEL_LENGTH-1]
+          else
+            text
+          end
+        end
+
+        def limit_number_of_labels(pairs)
+          if pairs.length > MAX_LABEL_COUNT
+            Mmtrix::Agent.logger.warn("Too many labels defined. Only taking first #{MAX_LABEL_COUNT}")
+            pairs[0...64]
+          else
+            pairs
+          end
+        end
+
+        # We only take the last value provided for a given label type key
+        def remove_duplicates(pairs)
+          grouped_by_type = pairs.group_by(&:first)
+          grouped_by_type.values.map(&:last)
+        end
+
+        def parse_labels_from_dictionary
+          make_label_hash(Mmtrix::Agent.config[:labels])
+        end
+
+        # Generally only useful during initial construction and tests
+        def reset_to_defaults
+          @high_security_source = nil
+          @environment_source   = EnvironmentSource.new
+          @server_source        = nil
+          @manual_source        = nil
+          @yaml_source          = nil
+          @default_source       = DefaultSource.new
+
+          @configs_for_testing  = []
+
+          reset_cache
+        end
+
+        def reset_cache
+          @cache = Hash.new {|hash,key| hash[key] = self.fetch(key) }
+        end
+
+        def log_config(direction, source)
+          # Just generating this log message (specifically calling
+          # flattened.inspect) is expensive enough that we don't want to do it
+          # unless we're actually going to be logging the message based on our
+          # current log level.
+          ::Mmtrix::Agent.logger.debug do
+            "Updating config (#{direction}) from #{source.class}. Results: #{flattened.inspect}"
+          end
+        end
+
+        def delete_all_configs_for_testing
+          @high_security_source = nil
+          @environment_source   = nil
+          @server_source        = nil
+          @manual_source        = nil
+          @yaml_source          = nil
+          @default_source       = nil
+          @configs_for_testing  = []
+        end
+
+        def num_configs_for_testing
+          config_stack.size
+        end
+
+        def config_classes_for_testing
+          config_stack.map(&:class)
+        end
+
+        private
+
+        def config_stack
+          stack = [@high_security_source,
+                   @environment_source,
+                   @server_source,
+                   @manual_source,
+                   @yaml_source,
+                   @default_source]
+
+          stack.compact!
+
+          @configs_for_testing.each do |config, at_start|
+            if at_start
+              stack.insert(0, config)
+            else
+              stack.push(config)
+            end
+          end
+
+          stack
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/configuration/manual_source.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/configuration/dotted_hash'
+
+module Mmtrix
+  module Agent
+    module Configuration
+      class ManualSource < DottedHash
+        def initialize(hash)
+          super(hash, true)
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/configuration/mask_defaults.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Configuration
+      MASK_DEFAULTS = {
+        :'thread_profiler' =>         Proc.new { !Mmtrix::Agent::Threading::BacktraceService.is_supported? },
+        :'thread_profiler.enabled' => Proc.new { !Mmtrix::Agent::Threading::BacktraceService.is_supported? }
+      }
+    end
+  end
+end