mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/datastores/mongo.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        def self.is_supported_version?
+          # No version constant in < 2.0 versions of Mongo :(
+          defined?(::Mongo) &&
+            defined?(::Mongo::MongoClient) &&
+            !is_version2?
+        end
+
+        # At present we explicitly don't support version 2.x of the driver yet
+        def self.is_version2?
+          defined?(::Mongo::VERSION) &&
+            Mmtrix::VersionNumber.new(::Mongo::VERSION) > Mmtrix::VersionNumber.new("2.0.0")
+        end
+
+        def self.is_version_1_10_or_later?
+          # Again, no VERSION constant in 1.x, so we have to rely on constant checks
+          defined?(::Mongo::CollectionOperationWriter)
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/datastores.rb

@@ -0,0 +1,191 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/datastores/metric_helper'
+
+module Mmtrix
+  module Agent
+    #
+    # This module contains helper methods to facilitate instrumentation of
+    # datastores not directly supported by the Ruby agent. It is intended to be
+    # primarily used by authors of 3rd-party datastore instrumentation.
+    #
+    # @api public
+    module Datastores
+
+      # @!group Tracing query methods
+
+      # Add Datastore tracing to a method. This properly generates the metrics
+      # for Mmtrix's Datastore features. It does not capture the actual
+      # query content into Transaction Traces. Use wrap if you want to provide
+      # that functionality.
+      #
+      # @param [Class] clazz the class to instrument
+      #
+      # @param [String, Symbol] method_name the name of instance method to
+      #   instrument
+      #
+      # @param [String] product name of your datastore for use in metric naming, e.g. "Redis"
+      #
+      # @param [optional,String] operation the name of operation if different
+      #   than the instrumented method name
+      #
+      # @api public
+      #
+      def self.trace(clazz, method_name, product, operation = method_name)
+        clazz.class_eval do
+          method_name_without_mmtrix = "#{method_name}_without_mmtrix"
+
+          if Mmtrix::Helper.instance_methods_include?(clazz, method_name) &&
+             !Mmtrix::Helper.instance_methods_include?(clazz, method_name_without_mmtrix)
+
+            visibility = Mmtrix::Helper.instance_method_visibility(clazz, method_name)
+
+            alias_method method_name_without_mmtrix, method_name
+
+            define_method(method_name) do |*args, &blk|
+              metrics = MetricHelper.metrics_for(product, operation)
+              Mmtrix::Agent::MethodTracer.trace_execution_scoped(metrics) do
+                send(method_name_without_mmtrix, *args, &blk)
+              end
+            end
+
+            send visibility, method_name
+            send visibility, method_name_without_mmtrix
+          end
+        end
+      end
+
+      # Wrap a call to a datastore and record Mmtrix Datastore metrics. This
+      # method can be used when a collection (i.e. table or model name) is
+      # known at runtime to be included in the metric naming. It is intended
+      # for situations that the simpler Mmtrix::Agent::Datastores.trace can't
+      # properly handle.
+      #
+      # To use this, wrap the datastore operation in the block passed to wrap.
+      #
+      #   Mmtrix::Agent::Datastores.wrap("FauxDB", "find", "items") do
+      #     FauxDB.find(query)
+      #   end
+      #
+      # @param [String] product the datastore name for use in metric naming,
+      #   e.g. "FauxDB"
+      #
+      # @param [String,Symbol] operation the name of operation (e.g. "select"),
+      #   often named after the method that's being instrumented.
+      #
+      # @param [optional, String] collection the collection name for use in
+      #   statement-level metrics (i.e. table or model name)
+      #
+      # @param [Proc,#call] callback proc or other callable to invoke after
+      #   running the datastore block. Receives three arguments: result of the
+      #   yield, the most specific (scoped) metric name, and elapsed time of the
+      #   call. An example use is attaching SQL to Transaction Traces at the end
+      #   of a wrapped datastore call.
+      #
+      #     callback = Proc.new do |result, metrics, elapsed|
+      #       Mmtrix::Agent::Datastores.notice_sql(query, metrics, elapsed)
+      #     end
+      #
+      #     Mmtrix::Agent::Datastores.wrap("FauxDB", "find", "items", callback) do
+      #       FauxDB.find(query)
+      #     end
+      #
+      # @note THERE ARE SECURITY CONCERNS WHEN CAPTURING QUERY TEXT!
+      #   Mmtrix's Transaction Tracing and Slow SQL features will
+      #   attempt to apply obfuscation to the passed queries, but it is possible
+      #   for a query format to be unsupported and result in exposing user
+      #   information embedded within captured queries.
+      #
+      # @api public
+      #
+      def self.wrap(product, operation, collection = nil, callback = nil)
+        return yield unless operation
+
+        metrics = MetricHelper.metrics_for(product, operation, collection)
+        scoped_metric = metrics.first
+        Mmtrix::Agent::MethodTracer.trace_execution_scoped(metrics) do
+          t0 = Time.now
+          begin
+            result = yield
+          ensure
+            if callback
+              elapsed_time = (Time.now - t0).to_f
+              callback.call(result, scoped_metric, elapsed_time)
+            end
+          end
+        end
+      end
+
+      # @!group Capturing query / statement text
+
+      # Wrapper for simplifying attaching SQL queries during a transaction.
+      #
+      # If you are recording non-SQL data, please use {notice_statement}
+      # instead.
+      #
+      #   Mmtrix::Agent::Datastores.notice_sql(query, metrics, elapsed)
+      #
+      # @param [String] query the SQL text to be captured. Note that depending
+      #   on user settings, this string will be run through obfuscation, but
+      #   some dialects of SQL (or non-SQL queries) are not guaranteed to be
+      #   properly obfuscated by these routines!
+      #
+      # @param [String] scoped_metric The most specific metric relating to this
+      #   query. Typically the result of
+      #   Mmtrix::Agent::Datastores::MetricHelper#metrics_for
+      #
+      # @param [Float] elapsed the elapsed time during query execution
+      #
+      # @note THERE ARE SECURITY CONCERNS WHEN CAPTURING QUERY TEXT!
+      #   Mmtrix's Transaction Tracing and Slow SQL features will
+      #   attempt to apply obfuscation to the passed queries, but it is possible
+      #   for a query format to be unsupported and result in exposing user
+      #   information embedded within captured queries.
+      #
+      # @api public
+      #
+      def self.notice_sql(query, scoped_metric, elapsed)
+        agent = Mmtrix::Agent.instance
+        agent.transaction_sampler.notice_sql(query, nil, elapsed)
+        agent.sql_sampler.notice_sql(query, scoped_metric, nil, elapsed)
+        nil
+      end
+
+      # Wrapper for simplifying attaching non-SQL data statements to a
+      # transaction. For instance, Mongo or CQL queries, Memcached or Redis
+      # keys would all be appropriate data to attach as statements.
+      #
+      # Data passed to this method is NOT obfuscated by Mmtrix, so please
+      # ensure that user information is obfuscated if the agent setting
+      # `transaction_tracer.record_sql` is set to `obfuscated`
+      #
+      #   Mmtrix::Agent::Datastores.notice_statement("key", elapsed)
+      #
+      # @param [String] statement text of the statement to capture.
+      #
+      # @param [Float] elapsed the elapsed time during query execution
+      #
+      # @note THERE ARE SECURITY CONCERNS WHEN CAPTURING STATEMENTS!
+      #   This method will properly ignore statements when the user has turned
+      #   off capturing queries, but it is not able to obfuscate arbitrary data!
+      #   To prevent exposing user information embedded in captured queries,
+      #   please ensure all data passed to this method is safe to transmit to
+      #   Mmtrix.
+      #
+      # @api public
+      #
+      def self.notice_statement(statement, elapsed)
+        # Settings may change eventually, but for now we follow the same
+        # capture rules as SQL for non-SQL statements.
+        return unless Mmtrix::Agent::Database.should_record_sql?
+
+        agent = Mmtrix::Agent.instance
+        agent.transaction_sampler.notice_nosql_statement(statement, elapsed)
+        nil
+      end
+
+    end
+  end
+end

data/lib/mmtrix/agent/deprecator.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Deprecator
+      def self.deprecate(method_name, new_method_name = nil, version = nil)
+        msgs = ["The method #{method_name} is deprecated."]
+        msgs << "It will be removed in version #{version}." if version
+        msgs << "Please use #{new_method_name} instead." if new_method_name
+
+        Mmtrix::Agent.logger.log_once(:warn, "deprecated_#{method_name}".to_sym, msgs)
+        Mmtrix::Agent.record_metric("Supportability/Deprecated/#{method_name}", 1)
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/encoding_normalizer.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This module was extracted from Mmtrix::JSONWrapper
+
+module Mmtrix
+  module Agent
+    module EncodingNormalizer
+      def self.normalize_string(raw_string)
+        @normalizer ||= choose_normalizer
+        @normalizer.normalize(raw_string)
+      end
+
+      def self.normalize_object(object)
+        case object
+        when String
+          normalize_string(object)
+        when Symbol
+          normalize_string(object.to_s)
+        when Array
+          return object if object.empty?
+          object.map { |x| normalize_object(x) }
+        when Hash
+          return object if object.empty?
+          hash = {}
+          object.each_pair do |k, v|
+            k = normalize_string(k)      if k.is_a?(String)
+            k = normalize_string(k.to_s) if k.is_a?(Symbol)
+            hash[k] = normalize_object(v)
+          end
+          hash
+        else
+          object
+        end
+      end
+
+      def self.choose_normalizer
+        if Mmtrix::LanguageSupport.supports_string_encodings?
+          EncodingNormalizer
+        else
+          IconvNormalizer
+        end
+      end
+
+      module EncodingNormalizer
+        def self.normalize(raw_string)
+          encoding = raw_string.encoding
+          if (encoding == Encoding::UTF_8 || encoding == Encoding::ISO_8859_1) && raw_string.valid_encoding?
+            return raw_string
+          end
+
+          # If the encoding is not valid, or it's ASCII-8BIT, we know conversion to
+          # UTF-8 is likely to fail, so treat it as ISO-8859-1 (byte-preserving).
+          normalized = raw_string.dup
+          if encoding == Encoding::ASCII_8BIT || !raw_string.valid_encoding?
+            normalized.force_encoding(Encoding::ISO_8859_1)
+          else
+            # Encoding is valid and non-binary, so it might be cleanly convertible
+            # to UTF-8. Give it a try and fall back to ISO-8859-1 if it fails.
+            begin
+              normalized.encode!(Encoding::UTF_8)
+            rescue
+              normalized.force_encoding(Encoding::ISO_8859_1)
+            end
+          end
+          normalized
+        end
+      end
+
+      module IconvNormalizer
+        def self.normalize(raw_string)
+          if @iconv.nil?
+            require 'iconv'
+            @iconv = Iconv.new('utf-8', 'iso-8859-1')
+          end
+          @iconv.iconv(raw_string)
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/error_collector.rb

@@ -0,0 +1,316 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    # This class collects errors from the parent application, storing
+    # them until they are harvested and transmitted to the server
+    class ErrorCollector
+      include Mmtrix::CollectionHelper
+
+      # Maximum possible length of the queue - defaults to 20, may be
+      # made configurable in the future. This is a tradeoff between
+      # memory and data retention
+      MAX_ERROR_QUEUE_LENGTH = 20 unless defined? MAX_ERROR_QUEUE_LENGTH
+      EXCEPTION_TAG_IVAR = :'@__nr_seen_exception' unless defined? EXCEPTION_TAG_IVAR
+
+      attr_accessor :errors
+
+      # Returns a new error collector
+      def initialize
+        @errors = []
+
+        # lookup of exception class names to ignore.  Hash for fast access
+        @ignore = {}
+
+        initialize_ignored_errors(Agent.config[:'error_collector.ignore_errors'])
+        @lock = Mutex.new
+
+        Agent.config.register_callback(:'error_collector.enabled') do |config_enabled|
+          ::Mmtrix::Agent.logger.debug "Errors will #{config_enabled ? '' : 'not '}be sent to the Mmtrix service."
+        end
+        Agent.config.register_callback(:'error_collector.ignore_errors') do |ignore_errors|
+          initialize_ignored_errors(ignore_errors)
+        end
+      end
+
+      def initialize_ignored_errors(ignore_errors)
+        @ignore.clear
+        ignore_errors = ignore_errors.split(",") if ignore_errors.is_a? String
+        ignore_errors.each { |error| error.strip! }
+        ignore(ignore_errors)
+      end
+
+      def enabled?
+        Agent.config[:'error_collector.enabled']
+      end
+
+      def disabled?
+        !enabled?
+      end
+
+      # We store the passed block in both an ivar on the class, and implicitly
+      # within the body of the ignore_filter_proc method intentionally here.
+      # The define_method trick is needed to get around the fact that users may
+      # call 'return' from within their filter blocks, which would otherwise
+      # result in a LocalJumpError.
+      #
+      # The raw block is also stored in an instance variable so that we can
+      # return it later in its original form.
+      #
+      # This is all done at the class level in order to avoid the case where
+      # the user sets up an ignore filter on one instance of the ErrorCollector,
+      # and then that instance subsequently gets discarded during agent startup.
+      # (For example, if the agent is initially disabled, and then gets enabled
+      # via a call to manual_start later on.)
+      #
+      def self.ignore_error_filter=(block)
+        @ignore_filter = block
+        if block
+          define_method(:ignore_filter_proc, &block)
+        elsif method_defined?(:ignore_filter_proc)
+          undef :ignore_filter_proc
+        end
+        @ignore_filter
+      end
+
+      def self.ignore_error_filter
+        @ignore_filter
+      end
+
+      # errors is an array of Exception Class Names
+      #
+      def ignore(errors)
+        errors.each do |error|
+          @ignore[error] = true
+          ::Mmtrix::Agent.logger.debug("Ignoring errors of type '#{error}'")
+        end
+      end
+
+      # Checks the provided error against the error filter, if there
+      # is an error filter
+      def filtered_by_error_filter?(error)
+        respond_to?(:ignore_filter_proc) && !ignore_filter_proc(error)
+      end
+
+      # Checks the array of error names and the error filter against
+      # the provided error
+      def filtered_error?(error)
+        @ignore[error.class.name] || filtered_by_error_filter?(error)
+      end
+
+      # an error is ignored if it is nil or if it is filtered
+      def error_is_ignored?(error)
+        error && filtered_error?(error)
+      rescue => e
+        Mmtrix::Agent.logger.error("Error '#{error}' will NOT be ignored. Exception '#{e}' while determining whether to ignore or not.", e)
+        false
+      end
+
+      # Calling instance_variable_set on a wrapped Java object in JRuby will
+      # generate a warning unless that object's class has already been marked
+      # as persistent, so we skip tagging of exception objects that are actually
+      # wrapped Java objects on JRuby.
+      #
+      # See https://github.com/jruby/jruby/wiki/Persistence
+      #
+      def exception_is_java_object?(exception)
+        Mmtrix::LanguageSupport.jruby? && exception.respond_to?(:java_class)
+      end
+
+      def exception_tagged?(exception)
+        return false if exception_is_java_object?(exception)
+        exception.instance_variable_get(EXCEPTION_TAG_IVAR)
+      end
+
+      def tag_exception(exception)
+        return if exception_is_java_object?(exception)
+        return if exception.frozen?
+        begin
+          exception.instance_variable_set(EXCEPTION_TAG_IVAR, true)
+        rescue => e
+          Mmtrix::Agent.logger.warn("Failed to tag exception: #{exception}: ", e)
+        end
+      end
+
+      def blamed_metric_name(txn, options)
+        if options[:metric] && options[:metric] != ::Mmtrix::Agent::UNKNOWN_METRIC
+          "Errors/#{options[:metric]}"
+        else
+          "Errors/#{txn.best_name}" if txn
+        end
+      end
+
+      def aggregated_metric_names(txn)
+        metric_names = ["Errors/all"]
+        return metric_names unless txn
+
+        if txn.recording_web_transaction?
+          metric_names << "Errors/allWeb"
+        else
+          metric_names << "Errors/allOther"
+        end
+
+        metric_names
+      end
+
+      # Increments a statistic that tracks total error rate
+      def increment_error_count!(state, exception, options={})
+        txn = state.current_transaction
+
+        metric_names  = aggregated_metric_names(txn)
+        blamed_metric = blamed_metric_name(txn, options)
+        metric_names << blamed_metric if blamed_metric
+
+        stats_engine = Mmtrix::Agent.agent.stats_engine
+        stats_engine.record_unscoped_metrics(state, metric_names) do |stats|
+          stats.increment_count
+        end
+      end
+
+      def skip_notice_error?(exception)
+        disabled? ||
+          error_is_ignored?(exception) ||
+          exception.nil? ||
+          exception_tagged?(exception)
+      end
+
+      # calls a method on an object, if it responds to it - used for
+      # detection and soft fail-safe. Returns nil if the method does
+      # not exist
+      def sense_method(object, method)
+        object.send(method) if object.respond_to?(method)
+      end
+
+      # extracts a stack trace from the exception for debugging purposes
+      def extract_stack_trace(exception)
+        actual_exception = sense_method(exception, 'original_exception') || exception
+        sense_method(actual_exception, 'backtrace') || '<no stack trace>'
+      end
+
+      # checks the size of the error queue to make sure we are under
+      # the maximum limit, and logs a warning if we are over the limit.
+      def over_queue_limit?(message)
+        over_limit = (@errors.reject{|err| err.is_internal}.length >= MAX_ERROR_QUEUE_LENGTH)
+        ::Mmtrix::Agent.logger.warn("The error reporting queue has reached #{MAX_ERROR_QUEUE_LENGTH}. The error detail for this and subsequent errors will not be transmitted to Mmtrix until the queued errors have been sent: #{message}") if over_limit
+        over_limit
+      end
+
+      # Synchronizes adding an error to the error queue, and checks if
+      # the error queue is too long - if so, we drop the error on the
+      # floor after logging a warning.
+      def add_to_error_queue(noticed_error)
+        @lock.synchronize do
+          if !over_queue_limit?(noticed_error.message) && !@errors.include?(noticed_error)
+            @errors << noticed_error
+          end
+        end
+      end
+
+      # See Mmtrix::Agent.notice_error for options and commentary
+      def notice_error(exception, options={}) #THREAD_LOCAL_ACCESS
+        return if skip_notice_error?(exception)
+
+        tag_exception(exception)
+
+        state = ::Mmtrix::Agent::TransactionState.tl_get
+        increment_error_count!(state, exception, options)
+        add_to_error_queue(create_noticed_error(exception, options))
+
+        exception
+      rescue => e
+        ::Mmtrix::Agent.logger.warn("Failure when capturing error '#{exception}':", e)
+      end
+
+      EMPTY_STRING = ''.freeze
+
+      def create_noticed_error(exception, options)
+        error_metric = options.delete(:metric) || EMPTY_STRING
+
+        noticed_error = Mmtrix::NoticedError.new(error_metric, exception)
+        noticed_error.request_uri = options.delete(:uri) || EMPTY_STRING
+        noticed_error.attributes  = options.delete(:attributes)
+
+        noticed_error.file_name   = sense_method(exception, :file_name)
+        noticed_error.line_number = sense_method(exception, :line_number)
+        noticed_error.stack_trace = extract_stack_trace(exception)
+
+        handle_deprecated_options(options)
+
+        noticed_error.attributes_from_notice_error = options.delete(:custom_params) || {}
+
+        # Any options that are passed to notice_error which aren't known keys
+        # get treated as custom attributes, so merge them into that hash.
+        noticed_error.attributes_from_notice_error.merge!(options)
+
+        noticed_error
+      end
+
+      DEPRECATED_OPTIONS_MSG = "Passing %s to notice_error is no longer supported. Set values on the enclosing transaction or record them as custom attributes instead.".freeze
+      DEPRECATED_OPTIONS = [:request_params, :request, :referer].freeze
+
+      # Old options no longer used by notice_error can still be passed. If they
+      # are, they shouldn't get merged into custom attributes. Delete and
+      # warn callers so they can fix their calls to notice_error.
+      def handle_deprecated_options(options)
+        DEPRECATED_OPTIONS.each do |deprecated|
+          if options.include?(deprecated)
+            Mmtrix::Agent.logger.warn(DEPRECATED_OPTIONS_MSG % deprecated)
+            options.delete(deprecated)
+          end
+        end
+      end
+
+      # *Use sparingly for difficult to track bugs.*
+      #
+      # Track internal agent errors for communication back to Mmtrix.
+      # To use, make a specific subclass of Mmtrix::Agent::InternalAgentError,
+      # then pass an instance of it to this method when your problem occurs.
+      #
+      # Limits are treated differently for these errors. We only gather one per
+      # class per harvest, disregarding (and not impacting) the app error queue
+      # limit.
+      def notice_agent_error(exception)
+        return unless exception.class < Mmtrix::Agent::InternalAgentError
+
+        # Log 'em all!
+        Mmtrix::Agent.logger.info(exception)
+
+        @lock.synchronize do
+          # Already seen this class once? Bail!
+          return if @errors.any? { |err| err.exception_class_name == exception.class.name }
+
+          trace = exception.backtrace || caller.dup
+          noticed_error = Mmtrix::NoticedError.new("Mmtrix/AgentError", exception)
+          noticed_error.stack_trace = trace
+          @errors << noticed_error
+        end
+      rescue => e
+        Mmtrix::Agent.logger.info("Unable to capture internal agent error due to an exception:", e)
+      end
+
+      def merge!(errors)
+        errors.each do |error|
+          add_to_error_queue(error)
+        end
+      end
+
+      # Get the errors currently queued up.  Unsent errors are left
+      # over from a previous unsuccessful attempt to send them to the server.
+      def harvest!
+        @lock.synchronize do
+          errors = @errors
+          @errors = []
+          errors
+        end
+      end
+
+      def reset!
+        @lock.synchronize do
+          @errors = []
+        end
+      end
+    end
+  end
+end