oneapm_rpm 1.1.0

data/lib/one_apm/agent/database/obfuscator.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+require 'one_apm/agent/database/obfuscation_helpers'
+
+module OneApm
+  module Agent
+    module Database
+      class Obfuscator
+        include Singleton
+        include ObfuscationHelpers
+
+        attr_reader :obfuscator
+
+        QUERY_TOO_LARGE_MESSAGE     = "Query too large (over 16k characters) to safely obfuscate"
+        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
+
+        def initialize
+          reset
+        end
+
+        def reset
+          @obfuscator = method(:default_sql_obfuscator)
+        end
+
+        # Sets the sql obfuscator used to clean up sql when sending it
+        # to the server. Possible types are:
+        #
+        # :before => sets the block to run before the existing
+        # obfuscators
+        #
+        # :after => sets the block to run after the existing
+        # obfuscator(s)
+        #
+        # :replace => removes the current obfuscator and replaces it
+        # with the provided block
+        def set_sql_obfuscator(type, &block)
+          if type == :before
+            @obfuscator = OneApm::ChainedCall.new(block, @obfuscator)
+          elsif type == :after
+            @obfuscator = OneApm::ChainedCall.new(@obfuscator, block)
+          elsif type == :replace
+            @obfuscator = block
+          else
+            fail "unknown sql_obfuscator type #{type}"
+          end
+        end
+
+        def default_sql_obfuscator(sql)
+          if sql[-3,3] == '...'
+            return QUERY_TOO_LARGE_MESSAGE
+          end
+
+          stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
+          obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
+
+          obfuscated = obfuscate_numeric_literals(stmt)
+
+          if obfuscate_double_quotes
+            obfuscated = obfuscate_quoted_literals(obfuscated)
+            obfuscated = remove_comments(obfuscated)
+            if contains_quotes?(obfuscated)
+              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
+            end
+          else
+            obfuscated = obfuscate_single_quote_literals(obfuscated)
+            obfuscated = remove_comments(obfuscated)
+            if contains_single_quotes?(obfuscated)
+              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
+            end
+          end
+
+
+          obfuscated.to_s # return back to a regular String
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/agent/database/postgres_explain_obfuscator.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+require 'one_apm/agent/database'
+require 'one_apm/agent/database/obfuscation_helpers'
+
+module OneApm
+  module Agent
+    module Database
+      module PostgresExplainObfuscator
+        extend self
+
+        # Note that this regex can't be shared with the ones in the
+        # Database::Obfuscator class because here we don't look for
+        # backslash-escaped strings (and those regexes are backwards).
+        QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
+        LABEL_LINE_REGEX     = /^([^:\n]*:\s+).*$/.freeze
+
+        def obfuscate(explain)
+          # First, we replace all single-quoted strings.
+          # This is necessary in order to deal with multi-line string constants
+          # embedded in the explain output.
+          #
+          # Note that we look for both single or double quotes but do not
+          # replace double quotes in order to avoid accidentally latching onto a
+          # single quote character embedded within a quoted identifier (such as
+          # a table name).
+          #
+          # Note also that we make no special provisions for backslash-escaped
+          # single quotes (\') because these are canonicalized to two single
+          # quotes ('') in the explain output.
+          explain.gsub!(QUOTED_STRINGS_REGEX) do |match|
+            match.start_with?('"') ? match : '?'
+          end
+
+          # Now, mask anything after the first colon (:).
+          # All parts of the query that can appear in the explain output are
+          # prefixed with "<label>: ", so we want to preserve the label, but
+          # remove the rest of the line.
+          explain.gsub!(LABEL_LINE_REGEX,   '\1?')
+          explain
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/agent/datastores.rb

@@ -0,0 +1,175 @@
+# encoding: utf-8
+
+require 'one_apm/agent/datastores/metric_helper'
+
+module OneApm
+  module Agent
+    module Datastores
+
+      # Add Datastore tracing to a method. This properly generates the metrics
+      # for OneApm's Datastore features. It does not capture the actual
+      # query content into Transaction Traces. Use wrap if you want to provide
+      # that functionality.
+      #
+      # @param [Class] clazz the class to instrument
+      #
+      # @param [String, Symbol] method_name the name of instance method to
+      # instrument
+      #
+      # @param [String] product name of your datastore for use in metric naming, e.g. "Redis"
+      #
+      # @param [optional,String] operation the name of operation if different
+      # than the instrumented method name
+      #
+      # @api public
+      #
+      def self.trace(clazz, method_name, product, operation = method_name)
+        clazz.class_eval do
+          method_name_without_oneapm = "#{method_name}_without_oneapm"
+
+          if OneApm::Helper.instance_methods_include?(clazz, method_name) &&
+             !OneApm::Helper.instance_methods_include?(clazz, method_name_without_oneapm)
+
+            visibility = OneApm::Helper.instance_method_visibility(clazz, method_name)
+
+            alias_method method_name_without_oneapm, method_name
+
+            define_method(method_name) do |*args, &blk|
+              metrics = MetricHelper.metrics_for(product, operation)
+              OneApm::Support::MethodTracer.trace_execution_scoped(metrics) do
+                send(method_name_without_oneapm, *args, &blk)
+              end
+            end
+
+            send visibility, method_name
+            send visibility, method_name_without_oneapm
+          end
+        end
+      end
+
+      # Wrap a call to a datastore and record OneApm Datastore metrics. This
+      # method can be used when a collection (i.e. table or model name) is
+      # known at runtime to be included in the metric naming. It is intended
+      # for situations that the simpler OneApm::Agent::Datastores.trace can't
+      # properly handle.
+      #
+      # To use this, wrap the datastore operation in the block passed to wrap.
+      #
+      #   OneApm::Agent::Datastores.wrap("FauxDB", "find", "items") do
+      #     FauxDB.find(query)
+      #   end
+      #
+      # @param [String] product the datastore name for use in metric naming,
+      # e.g. "FauxDB"
+      #
+      # @param [String,Symbol] operation the name of operation (e.g. "select"),
+      # often named after the method that's being instrumented.
+      #
+      # @param [optional, String] collection the collection name for use in
+      # statement-level metrics (i.e. table or model name)
+      #
+      # @param [Proc,#call] callback proc or other callable to invoke after
+      # running the datastore block. Receives three arguments: result of the
+      # yield, the most specific (scoped) metric name, and elapsed time of the
+      # call. An example use is attaching SQL to Transaction Traces at the end
+      # of a wrapped datastore call.
+      #
+      #   callback = Proc.new do |result, metrics, elapsed|
+      #     OneApm::Agent::Datastores.notice_sql(query, metrics, elapsed)
+      #   end
+      #
+      #   OneApm::Agent::Datastores.wrap("FauxDB", "find", "items", callback) do
+      #     FauxDB.find(query)
+      #   end
+      #
+      # **NOTE: THERE ARE SECURITY CONCERNS WHEN CAPTURING SQL!**
+      # OneApm's Transaction Tracing and Slow SQL features will
+      # attempt to apply obfuscation to the passed queries, but it is possible
+      # for a query format to be unsupported and result in exposing user
+      # information embedded within captured queries.
+      #
+      # @api public
+      #
+      def self.wrap(product, operation, collection = nil, callback = nil)
+        return yield unless operation
+
+        metrics = MetricHelper.metrics_for(product, operation, collection)
+        scoped_metric = metrics.first
+        OneApm::Support::MethodTracer.trace_execution_scoped(metrics) do
+          t0 = Time.now
+          begin
+            result = yield
+          ensure
+            if callback
+              elapsed_time = (Time.now - t0).to_f
+              callback.call(result, scoped_metric, elapsed_time)
+            end
+          end
+        end
+      end
+
+      # Wrapper for simplifying attaching SQL queries during a transaction.
+      #
+      # If you are recording non-SQL data, please use the notice_statement
+      # method instead.
+      #
+      #   OneApm::Agent::Datastores.notice_sql(query, metrics, elapsed)
+      #
+      # @param [String] query the SQL text to be captured. Note that depending
+      # on user settings, this string will be run through obfuscation, but
+      # some dialects of SQL (or non-SQL queries) are not guaranteed to be
+      # properly obfuscated by these routines!
+      #
+      # @param [String] scoped_metric The most specific metric relating to this
+      # query. Typically the result of
+      # OneApm::Agent::Datastores::MetricHelper#metrics_for
+      #
+      # @param [Float] elapsed the elapsed time during query execution
+      #
+      # **NOTE: THERE ARE SECURITY CONCERNS WHEN CAPTURING SQL!**
+      # OneApm's Transaction Tracing and Slow SQL features will
+      # attempt to apply obfuscation to the passed queries, but it is possible
+      # for a query format to be unsupported and result in exposing user
+      # information embedded within captured queries.
+      #
+      def self.notice_sql(query, scoped_metric, elapsed)
+        agent = OneApm::Agent.instance
+        agent.transaction_sampler.notice_sql(query, nil, elapsed)
+        agent.sql_sampler.notice_sql(query, scoped_metric, nil, elapsed)
+        nil
+      end
+
+      # Wrapper for simplifying attaching non-SQL data statements to a
+      # transaction. For instance, Mongo or CQL queries, Memcached or Redis
+      # keys would all be appropriate data to attach as statements.
+      #
+      # Data passed to this method is NOT obfuscated by OneApm, so please
+      # ensure that user information is obfuscated if the agent setting
+      # `transaction_tracer.record_sql` is set to `obfuscated`
+      #
+      #   OneApm::Agent::Datastores.notice_statement("key", elapsed)
+      #
+      # @param [String] statement text of the statement to capture.
+      #
+      # @param [Float] elapsed the elapsed time during query execution
+      #
+      # **NOTE: THERE ARE SECURITY CONCERNS WHEN CAPTURING STATEMENTS!**
+      # This method will properly ignore statements when the user has turned
+      # off capturing queries, but it is not able to obfuscate arbitrary data!
+      # To prevent exposing user information embedded in captured queries,
+      # please ensure all data passed to this method is safe to transmit to
+      # OneApm.
+      #
+      def self.notice_statement(statement, elapsed)
+        # Settings may change eventually, but for now we follow the same
+        # capture rules as SQL for non-SQL statements.
+        return unless OneApm::Agent::Database.should_record_sql?
+
+        agent = OneApm::Agent.instance
+        agent.transaction_sampler.notice_nosql_statement(statement, elapsed)
+        nil
+      end
+
+    end
+  end
+end

data/lib/one_apm/agent/datastores/metric_helper.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+
+module OneApm
+  module Agent
+    module Datastores
+      module MetricHelper
+        ROLLUP_METRIC        = "Datastore/all".freeze
+        WEB_ROLLUP_METRIC    = "Datastore/allWeb".freeze
+        OTHER_ROLLUP_METRIC  = "Datastore/allOther".freeze
+        DEFAULT_PRODUCT_NAME = "ActiveRecord".freeze
+        OTHER = "Other".freeze
+
+        ALL = "all".freeze
+        ALL_WEB = "allWeb".freeze
+        ALL_OTHER = "allOther".freeze
+
+        def self.statement_metric_for(product, collection, operation)
+          "Datastore/statement/#{product}/#{collection}/#{operation}"
+        end
+
+        def self.operation_metric_for(product, operation)
+          "Datastore/operation/#{product}/#{operation}"
+        end
+
+        def self.product_suffixed_rollup(product, suffix)
+          "Datastore/#{product}/#{suffix}"
+        end
+
+        def self.product_rollup(product)
+          "Datastore/#{product}/all"
+        end
+
+        def self.suffixed_rollup(suffix)
+          "Datastore/#{suffix}"
+        end
+
+        def self.all_suffix
+          if OneApm::Transaction.recording_web_transaction?
+            ALL_WEB
+          else
+            ALL_OTHER
+          end
+        end
+
+        def self.metrics_for(product, operation, collection = nil)
+          if overrides = overridden_operation_and_collection
+            operation, collection = overrides
+          end
+
+          suffix = all_suffix
+
+          # Order of these metrics matters--the first metric in the list will
+          # be treated as the scoped metric in a bunch of different cases.
+          metrics = [
+            operation_metric_for(product, operation),
+            product_suffixed_rollup(product, suffix),
+            product_rollup(product),
+            suffixed_rollup(suffix),
+            ROLLUP_METRIC
+          ]
+
+          metrics.unshift statement_metric_for(product, collection, operation) if collection
+
+          metrics
+        end
+
+        def self.metrics_from_sql(product, sql)
+          operation = OneApm::Agent::Database.parse_operation_from_query(sql) || OTHER
+          metrics_for(product, operation)
+        end
+
+        # Allow Transaction#with_database_metric_name to override our
+        # collection and operation
+        def self.overridden_operation_and_collection #THREAD_LOCAL_ACCESS
+          state = OneApm::TransactionState.tl_get
+          txn   = state.current_transaction
+          txn ? txn.instrumentation_state[:datastore_override] : nil
+        end
+
+      end
+    end
+  end
+end

data/lib/one_apm/agent/datastores/mongo.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+
+module OneApm
+  module Agent
+    module Datastores
+      module Mongo
+        def self.is_supported_version?
+          # No version constant in < 2.0 versions of Mongo :(
+          defined?(::Mongo) &&
+            defined?(::Mongo::MongoClient) &&
+            !is_version2?
+        end
+
+        # At present we explicitly don't support version 2.x of the driver yet
+        def self.is_version2?
+          defined?(::Mongo::VERSION) &&
+            OneApm::VersionNumber.new(::Mongo::VERSION) > OneApm::VersionNumber.new("2.0.0")
+        end
+
+        def self.is_version_1_10_or_later?
+          # Again, no VERSION constant in 1.x, so we have to rely on constant checks
+          defined?(::Mongo::CollectionOperationWriter)
+        end
+      end
+    end
+  end
+end

data/lib/one_apm/agent/datastores/mongo/metric_translator.rb

@@ -0,0 +1,189 @@
+# encoding: utf-8
+
+require 'one_apm/agent/datastores/mongo/obfuscator'
+require 'one_apm/agent/datastores/metric_helper'
+
+module OneApm
+  module Agent
+    module Datastores
+      module Mongo
+        module MetricTranslator
+          def self.metrics_for(name, payload)
+            payload ||= {}
+
+            if collection_in_selector?(payload)
+              command_key = command_key_from_selector(payload)
+              name        = get_name_from_selector(command_key, payload)
+              collection  = get_collection_from_selector(command_key, payload)
+            else
+              collection = payload[:collection]
+            end
+
+            # The 1.10.0 version of the mongo driver renamed 'remove' to
+            # 'delete', but for metric consistency with previous versions we
+            # want to keep it as 'remove'.
+            name = 'remove' if name.to_s == 'delete'
+
+            if self.find_one?(name, payload)
+              name = 'findOne'
+            elsif self.find_and_remove?(name, payload)
+              name = 'findAndRemove'
+            elsif self.find_and_modify?(name, payload)
+              name = 'findAndModify'
+            elsif self.create_indexes?(name, payload)
+              name = 'createIndexes'
+            elsif self.create_index?(name, payload)
+              name = 'createIndex'
+              collection = self.collection_name_from_index(payload)
+            elsif self.drop_indexes?(name, payload)
+              name = 'dropIndexes'
+            elsif self.drop_index?(name, payload)
+              name = 'dropIndex'
+            elsif self.re_index?(name, payload)
+              name = 'reIndex'
+            elsif self.group?(name, payload)
+              name = 'group'
+              collection = collection_name_from_group_selector(payload)
+            elsif self.rename_collection?(name, payload)
+              name = 'renameCollection'
+              collection = collection_name_from_rename_selector(payload)
+            end
+
+            build_metrics(name, collection)
+          rescue => e
+            OneApm::Agent.logger.debug("Failure during Mongo metric generation", e)
+            []
+          end
+
+          MONGO_PRODUCT_NAME = "MongoDB".freeze
+
+          def self.build_metrics(name, collection)
+            OneApm::Agent::Datastores::MetricHelper.metrics_for(MONGO_PRODUCT_NAME,
+                                                                  name,
+                                                                  collection)
+          end
+
+          def self.collection_in_selector?(payload)
+            payload[:collection] == '$cmd' && payload[:selector]
+          end
+
+          NAMES_IN_SELECTOR = [
+            :findandmodify,
+
+            "aggregate",
+            "count",
+            "group",
+            "mapreduce",
+
+            :distinct,
+
+            :createIndexes,
+            :deleteIndexes,
+            :reIndex,
+
+            :collstats,
+            :renameCollection,
+            :drop,
+          ]
+
+          def self.command_key_from_selector(payload)
+            selector = payload[:selector]
+            NAMES_IN_SELECTOR.find do |check_name|
+              selector.key?(check_name)
+            end
+          end
+
+          def self.get_name_from_selector(command_key, payload)
+            if command_key
+              command_key.to_sym
+            else
+              OneApm::Agent.increment_metric("Supportability/Mongo/UnknownCollection")
+              payload[:selector].first.first unless command_key
+            end
+          end
+
+          CMD_COLLECTION = "$cmd".freeze
+
+          def self.get_collection_from_selector(command_key, payload)
+            if command_key
+              payload[:selector][command_key]
+            else
+              OneApm::Agent.increment_metric("Supportability/Mongo/UnknownCollection")
+              CMD_COLLECTION
+            end
+          end
+
+          def self.find_one?(name, payload)
+            name == :find && payload[:limit] == -1
+          end
+
+          def self.find_and_modify?(name, payload)
+            name == :findandmodify
+          end
+
+          def self.find_and_remove?(name, payload)
+            name == :findandmodify && payload[:selector] && payload[:selector][:remove]
+          end
+
+          def self.create_indexes?(name, paylod)
+            name == :createIndexes
+          end
+
+          def self.create_index?(name, payload)
+            name == :insert && payload[:collection] == "system.indexes"
+          end
+
+          def self.drop_indexes?(name, payload)
+            name == :deleteIndexes && payload[:selector] && payload[:selector][:index] == "*"
+          end
+
+          def self.drop_index?(name, payload)
+            name == :deleteIndexes
+          end
+
+          def self.re_index?(name, payload)
+            name == :reIndex && payload[:selector] && payload[:selector][:reIndex]
+          end
+
+          def self.group?(name, payload)
+            name == :group
+          end
+
+          def self.rename_collection?(name, payload)
+            name == :renameCollection
+          end
+
+          def self.collection_name_from_index(payload)
+            if payload[:documents]
+              if payload[:documents].is_a?(Array)
+                # mongo gem versions pre 1.10.0
+                document = payload[:documents].first
+              else
+                # mongo gem versions 1.10.0 and later
+                document = payload[:documents]
+              end
+
+              if document && document[:ns]
+                return document[:ns].split('.').last
+              end
+            end
+
+            'system.indexes'
+          end
+
+          def self.collection_name_from_group_selector(payload)
+            payload[:selector]["group"]["ns"]
+          end
+
+          def self.collection_name_from_rename_selector(payload)
+            parts = payload[:selector][:renameCollection].split('.')
+            parts.shift
+            parts.join('.')
+          end
+
+        end
+
+      end
+    end
+  end
+end