oneapm_rpm 1.1.0

data/lib/one_apm/collector/stats_engine/metric_stats.rb

@@ -0,0 +1,243 @@
+# encoding: utf-8
+
+require 'one_apm/support/language_support'
+
+module OneApm
+  module Collector
+    class StatsEngine
+      # Handles methods related to actual Metric collection
+      module MetricStats
+        SCOPE_PLACEHOLDER = '__SCOPE__'.freeze
+
+        # Update the unscoped metrics given in metric_names.
+        # metric_names may be either a single name, or an array of names.
+        #
+        # This is an internal method, subject to change at any time. Client apps
+        # and gems should use the public API (OneApm::Agent.record_metric)
+        # instead.
+        #
+        # There are four ways to use this method:
+        #
+        # 1. With a numeric value, it will update the Stats objects associated
+        #    with the given metrics by calling record_data_point(value, aux).
+        #    aux will be treated in this case as the exclusive time associated
+        #    with the call being recorded.
+        #
+        # 2. With a value of :apdex_s, :apdex_t, or :apdex_f, it will treat the
+        #    associated stats as an Apdex metric, updating it to reflect the
+        #    occurrence of a transaction falling into the given category.
+        #    The aux value in this case should be the apdex threshold used in
+        #    bucketing the request.
+        #
+        # 3. If a block is given, value and aux will be ignored, and instead the
+        #    Stats object associated with each named unscoped metric will be
+        #    yielded to the block for customized update logic.
+        #
+        # 4. If value is a Stats instance, it will be merged into the Stats
+        #    associated with each named unscoped metric.
+        #
+        # If this method is called during a transaction, the metrics will be
+        # attached to the Transaction, and not merged into the global set until
+        # the end of the transaction.
+        #
+        # Otherwise, the metrics will be recorded directly into the global set
+        # of metrics, under a lock.
+        #
+        # @api private
+        #
+        def tl_record_unscoped_metrics(metric_names, value=nil, aux=nil, &blk)
+          state = OneApm::TransactionState.tl_get
+          record_unscoped_metrics(state, metric_names, value, aux, &blk)
+        end
+
+        def record_unscoped_metrics(state, metric_names, value=nil, aux=nil, &blk)
+          txn = state.current_transaction
+          if txn
+            txn.metrics.record_unscoped(metric_names, value, aux, &blk)
+          else
+            specs = coerce_to_metric_spec_array(metric_names, nil)
+            with_stats_lock do
+              @stats_hash.record(specs, value, aux, &blk)
+            end
+          end
+        end
+
+        # Like tl_record_unscoped_metrics, but records a scoped metric as well.
+        #
+        # This is an internal method, subject to change at any time. Client apps
+        # and gems should use the public API (OneApm::Agent.record_metric)
+        # instead.
+        #
+        # The given scoped_metric will be recoded as both a scoped *and* an
+        # unscoped metric. The summary_metrics will be recorded as unscoped
+        # metrics only.
+        #
+        # If called during a transaction, all metrics will be attached to the
+        # Transaction, and not merged into the global set of metrics until the
+        # end of the transaction.
+        #
+        # If called outside of a transaction, only the *unscoped* metrics will
+        # be recorded, directly into the global set of metrics (under a lock).
+        #
+        # @api private
+        #
+        def tl_record_scoped_and_unscoped_metrics(scoped_metric, summary_metrics=nil, value=nil, aux=nil, &blk)
+          state = OneApm::TransactionState.tl_get
+          record_scoped_and_unscoped_metrics(state, scoped_metric, summary_metrics, value, aux, &blk)
+        end
+
+        def record_scoped_and_unscoped_metrics(state, scoped_metric, summary_metrics=nil, value=nil, aux=nil, &blk)
+          txn = state.current_transaction
+          if txn
+            txn.metrics.record_scoped(scoped_metric, value, aux, &blk)
+            txn.metrics.record_unscoped(scoped_metric, value, aux, &blk)
+            if summary_metrics
+              txn.metrics.record_unscoped(summary_metrics, value, aux, &blk)
+            end
+          else
+            specs = coerce_to_metric_spec_array(scoped_metric, nil)
+            if summary_metrics
+              specs.concat(coerce_to_metric_spec_array(summary_metrics, nil))
+            end
+            with_stats_lock do
+              @stats_hash.record(specs, value, aux, &blk)
+            end
+          end
+        end
+
+        # This method is deprecated and not thread safe, and should not be used
+        # by any new client code. Use OneApm::Agent.record_metric instead.
+        #
+        # Lookup the Stats object for a given unscoped metric, returning a new
+        # Stats object if one did not exist previously.
+        #
+        # @api public
+        # @deprecated
+        #
+        def get_stats_no_scope(metric_name)
+          get_stats(metric_name, false)
+        end
+
+        # This method is deprecated and not thread safe, and should not be used
+        # by any new client code. Use OneApm::Agent.record_metric instead.
+        #
+        # If scoped_metric_only is true, only a scoped metric is created (used
+        # by rendering metrics which by definition are per controller only)
+        # Leaving second, unused parameter for compatibility
+        #
+        # @api public
+        # @deprecated
+        #
+        def get_stats(metric_name, _ = true, scoped_metric_only = false, scope = nil)
+          stats = nil
+          with_stats_lock do
+            if scoped_metric_only
+              stats = @stats_hash[OneApm::MetricSpec.new(metric_name, scope)]
+            else
+              unscoped_spec = OneApm::MetricSpec.new(metric_name)
+              unscoped_stats = @stats_hash[unscoped_spec]
+              if scope && scope != metric_name
+                scoped_spec = OneApm::MetricSpec.new(metric_name, scope)
+                scoped_stats = @stats_hash[scoped_spec]
+                stats = OneApm::Metrics::ChainedStats.new(scoped_stats, unscoped_stats)
+              else
+                stats = unscoped_stats
+              end
+            end
+          end
+          stats
+        end
+
+        # Returns a stat if one exists, otherwise returns nil.
+        def lookup_stats(metric_name, scope_name = '')
+          spec = OneApm::MetricSpec.new(metric_name, scope_name)
+          with_stats_lock do
+            @stats_hash.has_key?(spec) ? @stats_hash[spec] : nil
+          end
+        end
+
+        # Helper for recording a straight value into the count
+        def tl_record_supportability_metric_count(metric, value)
+          real_name = "Supportability/#{metric}"
+          tl_record_unscoped_metrics(real_name) do |stat|
+            stat.call_count = value
+          end
+        end
+
+        def reset!
+          with_stats_lock do
+            old = @stats_hash
+            @stats_hash = StatsHash.new
+            old
+          end
+        end
+
+        # merge data from previous harvests into this stats engine
+        def merge!(other_stats_hash)
+          with_stats_lock do
+            @stats_hash.merge!(other_stats_hash)
+            @stats_hash
+          end
+        end
+
+        def merge_transaction_metrics!(txn_metrics, scope)
+          with_stats_lock do
+            @stats_hash.merge_transaction_metrics!(txn_metrics, scope)
+          end
+        end
+
+        def harvest!
+          now = Time.now
+          snapshot = reset!
+          snapshot = apply_rules_to_metric_data(@metric_rules, snapshot)
+          snapshot.harvested_at = now
+          snapshot
+        end
+
+        def apply_rules_to_metric_data(rules_engine, stats_hash)
+          renamed_stats = OneApm::Collector::StatsHash.new(stats_hash.started_at)
+          stats_hash.each do |spec, stats|
+            new_name = rules_engine.rename(spec.name)
+            unless new_name.nil?
+              new_spec = OneApm::MetricSpec.new(new_name, spec.scope)
+              renamed_stats[new_spec].merge!(stats)
+            end
+          end
+          renamed_stats
+        end
+
+        def coerce_to_metric_spec_array(metric_names_or_specs, scope)
+          specs = []
+          Array(metric_names_or_specs).map do |name_or_spec|
+            case name_or_spec
+            when String
+              specs << OneApm::MetricSpec.new(name_or_spec)
+              specs << OneApm::MetricSpec.new(name_or_spec, scope) if scope
+            when OneApm::MetricSpec
+              specs << name_or_spec
+            end
+          end
+          specs
+        end
+
+        # For use by test code only.
+        def clear_stats
+          reset!
+          OneApm::Agent::BusyCalculator.reset
+        end
+
+        # Returns all of the metric names of all the stats in the engine.
+        # For use by test code only.
+        def metrics
+          with_stats_lock do
+            @stats_hash.keys.map { |spec| spec.to_s }
+          end
+        end
+
+        def metric_specs
+          with_stats_lock { @stats_hash.keys }
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/collector/stats_engine/stats_hash.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+
+# A Hash-descended class for storing metric data in the OneApm Agent.
+#
+# Keys are OneApm::MetricSpec objects.
+# Values are OneApm::Metrics::Stats objects.
+#
+# Missing keys will be automatically created as empty OneApm::Metrics::Stats
+# instances, so use has_key? explicitly to check for key existence.
+#
+# Note that instances of this class are intended to be append-only with respect
+# to new metrics. That is, you should not attempt to *remove* an entry after it
+# has been added, only update it (create a new instance if you need to start
+# over with a blank slate).
+#
+# This class makes no provisions for safe usage from multiple threads, such
+# measures should be externally provided.
+
+require 'one_apm/errors/internal_agent_error'
+
+module OneApm
+  module Collector
+    class StatsHash < ::Hash
+
+      attr_accessor :started_at, :harvested_at
+
+      def initialize(started_at=Time.now)
+        @started_at = started_at.to_f
+        super() { |hash, key| hash[key] = OneApm::Metrics::Stats.new }
+      end
+
+      def marshal_dump
+        [@started_at, Hash[self]]
+      end
+
+      def marshal_load(data)
+        @started_at = data.shift
+        self.merge!(data.shift)
+      end
+
+      def ==(other)
+        Hash[self] == Hash[other]
+      end
+
+      class StatsHashLookupError < OneApm::Agent::InternalAgentError
+        def initialize(original_error, hash, metric_spec)
+          super("Lookup error in StatsHash: #{original_error.class}: #{original_error.message}. Falling back adding #{metric_spec.inspect}")
+        end
+      end
+
+      def record(metric_specs, value=nil, aux=nil, &blk)
+        Array(metric_specs).each do |metric_spec|
+          stats = nil
+          begin
+            stats = self[metric_spec]
+          rescue NoMethodError => e
+            # This only happen in the case of a corrupted default_proc
+            # Side-step it manually, notice the issue, and carry on....
+            OneApm::Agent.instance.error_collector. \
+              notice_agent_error(StatsHashLookupError.new(e, self, metric_spec))
+
+            stats = OneApm::Metrics::Stats.new
+            self[metric_spec] = stats
+
+            # Try to restore the default_proc so we won't continually trip the error
+            if respond_to?(:default_proc=)
+              self.default_proc = Proc.new { |hash, key| hash[key] = OneApm::Metrics::Stats.new }
+            end
+          end
+
+          stats.record(value, aux, &blk)
+        end
+      end
+
+      def merge!(other)
+        if other.is_a?(StatsHash) && other.started_at < @started_at
+          @started_at = other.started_at
+        end
+        other.each do |key, val|
+          merge_or_insert(key, val)
+        end
+        self
+      end
+
+      def merge_transaction_metrics!(txn_metrics, scope)
+        txn_metrics.each_unscoped do |name, stats|
+          spec = OneApm::MetricSpec.new(name)
+          merge_or_insert(spec, stats)
+        end
+        txn_metrics.each_scoped do |name, stats|
+          spec = OneApm::MetricSpec.new(name, scope)
+          merge_or_insert(spec, stats)
+        end
+      end
+
+      def merge_or_insert(metric_spec, stats)
+        if self.has_key?(metric_spec)
+          self[metric_spec].merge!(stats)
+        else
+          self[metric_spec] = stats
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/configuration.rb

@@ -0,0 +1,429 @@
+# encoding: utf-8
+
+require 'forwardable'
+require 'one_apm/support/dotted_hash'
+require 'one_apm/configuration/yaml_source'
+require 'one_apm/configuration/server_source'
+require 'one_apm/configuration/manual_source'
+require 'one_apm/configuration/default_source'
+require 'one_apm/configuration/environment_source'
+require 'one_apm/configuration/high_security_source'
+
+module OneApm
+  module Configuration
+
+    MASK_DEFAULTS = {
+      :'thread_profiler' =>         Proc.new { !OneApm::Agent::Threading::BacktraceService.is_supported? },
+      :'thread_profiler.enabled' => Proc.new { !OneApm::Agent::Threading::BacktraceService.is_supported? }
+    }
+
+    class Manager
+
+      attr_reader :stripped_exceptions_whitelist
+
+      def initialize
+        reset_to_defaults
+        @callbacks = Hash.new {|hash,key| hash[key] = [] }
+
+        register_callback(:'strip_exception_messages.whitelist') do |whitelist|
+          if whitelist
+            @stripped_exceptions_whitelist = parse_constant_list(whitelist).compact
+          else
+            @stripped_exceptions_whitelist = []
+          end
+        end
+      end
+
+      # Defining these explicitly saves object allocations that we incur
+      # if we use Forwardable and def_delegators.
+      def [](key)
+        @cache[key]
+      end
+
+      def has_key?(key)
+        @cache.has_key?[key]
+      end
+
+      def keys
+        @cache.keys
+      end
+
+      def add_config_for_testing(source, level=0)
+        raise 'Invalid config type for testing' unless [Hash, OneApm::Support::DottedHash].include?(source.class)
+        invoke_callbacks(:add, source)
+        @configs_for_testing << [source.freeze, level]
+        reset_cache
+        log_config(:add, source)
+      end
+
+      def remove_config_type(sym)
+        source = case sym
+        when :high_security then @high_security_source
+        when :environment   then @environment_source
+        when :server        then @server_source
+        when :manual        then @manual_source
+        when :yaml          then @yaml_source
+        when :default       then @default_source
+        end
+
+        remove_config(source)
+      end
+
+      def remove_config(source)
+        case source
+        when HighSecuritySource then @high_security_source = nil
+        when EnvironmentSource  then @environment_source   = nil
+        when ServerSource       then @server_source        = nil
+        when ManualSource       then @manual_source        = nil
+        when YamlSource         then @yaml_source          = nil
+        when DefaultSource      then @default_source       = nil
+        else
+          @configs_for_testing.delete_if {|src,lvl| src == source}
+        end
+
+        reset_cache
+        invoke_callbacks(:remove, source)
+        log_config(:remove, source)
+      end
+
+      def replace_or_add_config(source)
+        source.freeze
+        was_finished = finished_configuring?
+
+        invoke_callbacks(:add, source)
+        case source
+        when HighSecuritySource then @high_security_source = source
+        when EnvironmentSource  then @environment_source   = source
+        when ServerSource       then @server_source        = source
+        when ManualSource       then @manual_source        = source
+        when YamlSource         then @yaml_source          = source
+        when DefaultSource      then @default_source       = source
+        else
+          OneApm::Agent.logger.warn("Invalid config format; config will be ignored: #{source}")
+        end
+
+        reset_cache
+        log_config(:add, source)
+
+        notify_finished_configuring if !was_finished && finished_configuring?
+      end
+
+      def source(key)
+        config_stack.each do |config|
+          if config.respond_to?(key.to_sym) || config.has_key?(key.to_sym)
+            return config
+          end
+        end
+      end
+
+      def fetch(key)
+        config_stack.each do |config|
+          next unless config
+          accessor = key.to_sym
+
+          if config.has_key?(accessor)
+            evaluated = evaluate_procs(config[accessor])
+
+            begin
+              return apply_transformations(accessor, evaluated)
+            rescue
+              next
+            end
+          end
+        end
+
+        nil
+      end
+
+      def evaluate_procs(value)
+        if value.respond_to?(:call)
+          instance_eval(&value)
+        else
+          value
+        end
+      end
+
+      def apply_transformations(key, value)
+        if transform = transform_from_default(key)
+          begin
+            transform.call(value)
+          rescue => e
+            ::OneApm::Agent.logger.error("Error applying transformation for #{key}, falling back to #{value}.", e)
+            raise e
+          end
+        else
+          value
+        end
+      end
+
+      def transform_from_default(key)
+        ::OneApm::Configuration::DefaultSource.transform_for(key)
+      end
+
+      def register_callback(key, &proc)
+        @callbacks[key] << proc
+        proc.call(@cache[key])
+      end
+
+      def invoke_callbacks(direction, source)
+        return unless source
+        source.keys.each do |key|
+
+          if @cache[key] != source[key]
+            @callbacks[key].each do |proc|
+              if direction == :add
+                proc.call(source[key])
+              else
+                proc.call(@cache[key])
+              end
+            end
+          end
+        end
+      end
+
+      def notify_finished_configuring
+        OneApm::Agent.instance.events.notify(:finished_configuring)
+      end
+
+      def finished_configuring?
+        !@server_source.nil?
+      end
+
+      def flattened
+        config_stack.reverse.inject({}) do |flat,layer|
+          thawed_layer = layer.to_hash.dup
+          thawed_layer.each do |k,v|
+            begin
+              thawed_layer[k] = instance_eval(&v) if v.respond_to?(:call)
+            rescue => e
+              ::OneApm::Agent.logger.debug("#{e.class.name} : #{e.message} - when accessing config key #{k}")
+              thawed_layer[k] = nil
+            end
+            thawed_layer.delete(:config)
+          end
+          flat.merge(thawed_layer.to_hash)
+        end
+      end
+
+      def apply_mask(hash)
+        MASK_DEFAULTS. \
+          select {|_, proc| proc.call}. \
+          each {|key, _| hash.delete(key) }
+        hash
+      end
+
+      def to_collector_hash
+        OneApm::Support::DottedHash.new(apply_mask(flattened)).to_hash.delete_if do |k, v|
+          default = DEFAULTS[k]
+          if default
+            default[:local_only]
+          else
+            # In our tests, we add totally bogus configs, because testing.
+            # In those cases, there will be no default. So we'll just let
+            # them through.
+            false
+          end
+        end
+      end
+
+      def app_names
+        case OneApm::Agent.config[:app_name]
+        when Array then OneApm::Agent.config[:app_name]
+        when String then OneApm::Agent.config[:app_name].split(';')
+        else []
+        end
+      end
+
+      MALFORMED_LABELS_WARNING = "Skipping malformed labels configuration"
+      PARSING_LABELS_FAILURE   = "Failure during parsing labels. Ignoring and carrying on with connect."
+
+      MAX_LABEL_COUNT  = 64
+      MAX_LABEL_LENGTH = 255
+
+      def parsed_labels
+        case OneApm::Agent.config[:labels]
+        when String
+          parse_labels_from_string
+        else
+          parse_labels_from_dictionary
+        end
+      rescue => e
+        OneApm::Agent.logger.error(PARSING_LABELS_FAILURE, e)
+        []
+      end
+
+      def parse_labels_from_string
+        labels = OneApm::Agent.config[:labels]
+        label_pairs = break_label_string_into_pairs(labels)
+        make_label_hash(label_pairs, labels)
+      end
+
+      def break_label_string_into_pairs(labels)
+        stripped_labels = labels.strip
+        stripped_labels.split(';').map do |pair|
+          pair.split(':').map(&:strip)
+        end
+      end
+
+      def valid_label_pairs?(label_pairs)
+        label_pairs.all? do |pair|
+          pair.length == 2 && valid_label_item?(pair.first) && valid_label_item?(pair.last)
+        end
+      end
+
+      def valid_label_item?(item)
+        case item
+        when String  then !item.empty?
+        when Numeric then true
+        when true    then true
+        when false   then true
+        else false
+        end
+      end
+
+      def make_label_hash(pairs, labels = nil)
+        # This can accept a hash, so force it down to an array of pairs first
+        pairs = Array(pairs)
+
+        unless valid_label_pairs?(pairs)
+          OneApm::Agent.logger.warn("#{MALFORMED_LABELS_WARNING}: #{labels||pairs}")
+          return []
+        end
+
+        pairs = limit_number_of_labels(pairs)
+        pairs = remove_duplicates(pairs)
+        pairs.map do |key, value|
+          {
+            'label_type'  => truncate(key),
+            'label_value' => truncate(value.to_s, key)
+          }
+        end
+      end
+
+      def truncate(text, key=nil)
+        if text.length > MAX_LABEL_LENGTH
+          if key
+            msg = "The value for the label '#{key}' is longer than the allowed #{MAX_LABEL_LENGTH} and will be truncated. Value = '#{text}'"
+          else
+            msg = "Label name longer than the allowed #{MAX_LABEL_LENGTH} will be truncated. Name = '#{text}'"
+          end
+          OneApm::Agent.logger.warn(msg)
+          text[0..MAX_LABEL_LENGTH-1]
+        else
+          text
+        end
+      end
+
+      def limit_number_of_labels(pairs)
+        if pairs.length > MAX_LABEL_COUNT
+          OneApm::Agent.logger.warn("Too many labels defined. Only taking first #{MAX_LABEL_COUNT}")
+          pairs[0...64]
+        else
+          pairs
+        end
+      end
+
+      # We only take the last value provided for a given label type key
+      def remove_duplicates(pairs)
+        grouped_by_type = pairs.group_by(&:first)
+        grouped_by_type.values.map(&:last)
+      end
+
+      def parse_labels_from_dictionary
+        make_label_hash(OneApm::Agent.config[:labels])
+      end
+
+      # Generally only useful during initial construction and tests
+      def reset_to_defaults
+        @high_security_source = nil
+        @environment_source   = EnvironmentSource.new
+        @server_source        = nil
+        @manual_source        = nil
+        @yaml_source          = nil
+        @default_source       = DefaultSource.new
+
+        @configs_for_testing  = []
+
+        reset_cache
+      end
+
+      def reset_cache
+        @cache = Hash.new {|hash,key| hash[key] = self.fetch(key) }
+      end
+
+      def log_config(direction, source)
+        # Just generating this log message (specifically calling
+        # flattened.inspect) is expensive enough that we don't want to do it
+        # unless we're actually going to be logging the message based on our
+        # current log level.
+        ::OneApm::Agent.logger.debug do
+          "Updating config (#{direction}) from #{source.class}. Results: #{flattened.inspect}"
+        end
+      end
+
+      def delete_all_configs_for_testing
+        @high_security_source = nil
+        @environment_source   = nil
+        @server_source        = nil
+        @manual_source        = nil
+        @yaml_source          = nil
+        @default_source       = nil
+        @configs_for_testing  = []
+      end
+
+      def num_configs_for_testing
+        config_stack.size
+      end
+
+      def config_classes_for_testing
+        config_stack.map(&:class)
+      end
+
+      private
+
+      def config_stack
+        stack = [
+          @high_security_source,
+          @environment_source,
+          @server_source,
+          @manual_source,
+          @yaml_source,
+          @default_source
+        ]
+
+        stack.compact!
+
+        @configs_for_testing.each do |config, at_start|
+          if at_start
+            stack.insert(0, config)
+          else
+            stack.push(config)
+          end
+        end
+
+        stack
+      end
+
+      def parse_constant_list(list)
+        list.split(/\s*,\s*/).map do |class_name|
+          const = constantize(class_name)
+
+          unless const
+            OneApm::Agent.logger.warn "Configuration referenced undefined constant: #{class_name}"
+          end
+
+          const
+        end
+      end
+
+      def constantize(class_name)
+        namespaces = class_name.split('::')
+
+        namespaces.inject(Object) do |namespace, name|
+          return unless namespace
+          namespace.const_get(name) if namespace.const_defined?(name)
+        end
+      end
+    end
+  end
+end