mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/database.rb

@@ -0,0 +1,325 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'singleton'
+require 'mmtrix/agent/database/obfuscation_helpers'
+require 'mmtrix/agent/database/obfuscator'
+require 'mmtrix/agent/database/postgres_explain_obfuscator'
+
+module Mmtrix
+  # columns for a mysql explain plan
+  MYSQL_EXPLAIN_COLUMNS = [
+                           "Id",
+                           "Select Type",
+                           "Table",
+                           "Type",
+                           "Possible Keys",
+                           "Key",
+                           "Key Length",
+                           "Ref",
+                           "Rows",
+                           "Extra"
+                          ].freeze
+
+  module Agent
+    module Database
+      MAX_QUERY_LENGTH = 16384
+
+      extend self
+
+      def capture_query(query)
+        Helper.correctly_encoded(truncate_query(query))
+      end
+
+      def truncate_query(query)
+        if query.length > (MAX_QUERY_LENGTH - 4)
+          query[0..MAX_QUERY_LENGTH - 4] + '...'
+        else
+          query
+        end
+      end
+
+      def obfuscate_sql(sql)
+        Obfuscator.instance.obfuscator.call(sql)
+      end
+
+      def set_sql_obfuscator(type, &block)
+        Obfuscator.instance.set_sql_obfuscator(type, &block)
+      end
+
+      def record_sql_method(config_section=:transaction_tracer)
+        key = record_sql_method_key(config_section)
+
+        case Agent.config[key].to_s
+        when 'off'
+          :off
+        when 'none'
+          :off
+        when 'false'
+          :off
+        when 'raw'
+          :raw
+        else
+          :obfuscated
+        end
+      end
+
+      def record_sql_method_key(config_section)
+        case config_section
+        when :transaction_tracer
+          :'transaction_tracer.record_sql'
+        when :slow_sql
+          :'slow_sql.record_sql'
+        else
+          "#{config_section}.record_sql".to_sym
+        end
+      end
+
+      RECORD_FOR = [:raw, :obfuscated].freeze
+
+      def should_record_sql?(config_section=:transaction_tracer)
+        RECORD_FOR.include?(record_sql_method(config_section))
+      end
+
+      def should_collect_explain_plans?(config_section=:transaction_tracer)
+        should_record_sql?(config_section) &&
+          Agent.config["#{config_section}.explain_enabled".to_sym]
+      end
+
+      def get_connection(config, &connector)
+        ConnectionManager.instance.get_connection(config, &connector)
+      end
+
+      def close_connections
+        ConnectionManager.instance.close_connections
+      end
+
+      # This takes a connection config hash from ActiveRecord or Sequel and
+      # returns a string describing the associated database adapter
+      def adapter_from_config(config)
+        if config[:adapter]
+          return config[:adapter].to_s
+        elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
+          # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
+          # jdbc-sqlite3 gems.
+          return $1
+        end
+      end
+
+      # Perform this in the runtime environment of a managed
+      # application, to explain the sql statement executed within a
+      # node of a transaction sample. Returns an array of
+      # explanations (which is an array rows consisting of an array of
+      # strings for each column returned by the the explain query)
+      # Note this happens only for statements whose execution time
+      # exceeds a threshold (e.g. 500ms) and only within the slowest
+      # transaction in a report period, selected for shipment to New
+      # Mmtrix
+      def explain_sql(sql, connection_config, explainer=nil)
+        return nil unless sql && explainer && connection_config
+        statement = sql.split(";\n")[0] # only explain the first
+        explain_plan = explain_statement(statement, connection_config, explainer)
+        return explain_plan || []
+      end
+
+      SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
+
+      def explain_statement(statement, config, explainer)
+        return unless explainer && is_select?(statement)
+
+        if statement[-3,3] == '...'
+          Mmtrix::Agent.logger.debug('Unable to collect explain plan for truncated query.')
+          return
+        end
+
+        if parameterized?(statement)
+          Mmtrix::Agent.logger.debug('Unable to collect explain plan for parameterized query.')
+          return
+        end
+
+        adapter = adapter_from_config(config)
+        if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
+          Mmtrix::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
+          return
+        end
+
+        handle_exception_in_explain do
+          start = Time.now
+          plan = explainer.call(config, statement)
+          ::Mmtrix::Agent.record_metric("Supportability/Database/execute_explain_plan", Time.now - start)
+          return process_resultset(plan, adapter) if plan
+        end
+      end
+
+      def process_resultset(results, adapter)
+        case adapter.to_s
+        when 'postgres', 'postgresql'
+          process_explain_results_postgres(results)
+        when 'mysql2'
+          process_explain_results_mysql2(results)
+        when 'mysql'
+          process_explain_results_mysql(results)
+        when 'sqlite'
+          process_explain_results_sqlite(results)
+        end
+      end
+
+      QUERY_PLAN = 'QUERY PLAN'.freeze
+
+      def process_explain_results_postgres(results)
+        if results.is_a?(String)
+          query_plan_string = results
+        else
+          lines = []
+          results.each { |row| lines << row[QUERY_PLAN] }
+          query_plan_string = lines.join("\n")
+        end
+
+        unless record_sql_method == :raw
+          query_plan_string = Mmtrix::Agent::Database::PostgresExplainObfuscator.obfuscate(query_plan_string)
+        end
+        values = query_plan_string.split("\n").map { |line| [line] }
+
+        [[QUERY_PLAN], values]
+      end
+
+      # Sequel returns explain plans as just one big pre-formatted String
+      # In that case, we send a nil headers array, and the single string
+      # wrapped in an array for the values.
+      # Note that we don't use this method for Postgres explain plans, since
+      # they need to be passed through the explain plan obfuscator first.
+      def string_explain_plan_results(results)
+        [nil, [results]]
+      end
+
+      def process_explain_results_mysql(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
+        headers = []
+        values  = []
+        if results.is_a?(Array)
+          # We're probably using the jdbc-mysql gem for JRuby, which will give
+          # us an array of hashes.
+          headers = results.first.keys
+          results.each do |row|
+            values << headers.map { |h| row[h] }
+          end
+        else
+          # We're probably using the native mysql driver gem, which will give us
+          # a Mysql::Result object that responds to each_hash
+          results.each_hash do |row|
+            headers = row.keys
+            values << headers.map { |h| row[h] }
+          end
+        end
+        [headers, values]
+      end
+
+      def process_explain_results_mysql2(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
+        headers = results.fields
+        values  = []
+        results.each { |row| values << row }
+        [headers, values]
+      end
+
+      SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
+
+      def process_explain_results_sqlite(results)
+        return string_explain_plan_results(results) if results.is_a?(String)
+        headers = SQLITE_EXPLAIN_COLUMNS
+        values  = []
+        results.each do |row|
+          values << headers.map { |h| row[h] }
+        end
+        [headers, values]
+      end
+
+      def handle_exception_in_explain
+        yield
+      rescue => e
+        begin
+          # guarantees no throw from explain_sql
+          ::Mmtrix::Agent.logger.error("Error getting query plan:", e)
+          nil
+        rescue
+          # double exception. throw up your hands
+          nil
+        end
+      end
+
+      KNOWN_OPERATIONS = [
+        'alter',
+        'select',
+        'update',
+        'delete',
+        'insert',
+        'create',
+        'show',
+        'set',
+        'exec',
+        'execute',
+        'call'
+      ]
+
+      SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
+      EMPTY_STRING      = ''.freeze
+
+      def parse_operation_from_query(sql)
+        sql = sql.gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
+        if sql =~ /(\w+)/
+          op = $1.downcase
+          return op if KNOWN_OPERATIONS.include?(op)
+        end
+      end
+
+      def is_select?(statement)
+        parse_operation_from_query(statement) == 'select'
+      end
+
+      def parameterized?(statement)
+        Obfuscator.instance.obfuscate_single_quote_literals(statement) =~ /\$\d+/
+      end
+
+      class ConnectionManager
+        include Singleton
+
+        # Returns a cached connection for a given ActiveRecord
+        # configuration - these are stored or reopened as needed, and if
+        # we cannot get one, we ignore it and move on without explaining
+        # the sql
+        def get_connection(config, &connector)
+          @connections ||= {}
+
+          connection = @connections[config]
+
+          return connection if connection
+
+          begin
+            @connections[config] = connector.call(config)
+          rescue => e
+            ::Mmtrix::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
+            nil
+          end
+        end
+
+        # Closes all the connections in the internal connection cache
+        def close_connections
+          @connections ||= {}
+          @connections.values.each do |connection|
+            begin
+              connection.disconnect!
+            rescue
+            end
+          end
+
+          @connections = {}
+        end
+      end
+
+      class Statement < String
+        attr_accessor :adapter, :config, :explainer
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/datastores/metric_helper.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module MetricHelper
+        ROLLUP_METRIC        = "Datastore/all".freeze
+        WEB_ROLLUP_METRIC    = "Datastore/allWeb".freeze
+        OTHER_ROLLUP_METRIC  = "Datastore/allOther".freeze
+        DEFAULT_PRODUCT_NAME = "ActiveRecord".freeze
+        OTHER = "Other".freeze
+
+        ALL = "all".freeze
+        ALL_WEB = "allWeb".freeze
+        ALL_OTHER = "allOther".freeze
+
+        def self.statement_metric_for(product, collection, operation)
+          "Datastore/statement/#{product}/#{collection}/#{operation}"
+        end
+
+        def self.operation_metric_for(product, operation)
+          "Datastore/operation/#{product}/#{operation}"
+        end
+
+        def self.product_suffixed_rollup(product, suffix)
+          "Datastore/#{product}/#{suffix}"
+        end
+
+        def self.product_rollup(product)
+          "Datastore/#{product}/all"
+        end
+
+        def self.suffixed_rollup(suffix)
+          "Datastore/#{suffix}"
+        end
+
+        def self.all_suffix
+          if Mmtrix::Agent::Transaction.recording_web_transaction?
+            ALL_WEB
+          else
+            ALL_OTHER
+          end
+        end
+
+        def self.metrics_for(product, operation, collection = nil, generic_product = nil)
+          if overrides = overridden_operation_and_collection
+            if should_override?(overrides, product, generic_product)
+              operation  = overrides[0] || operation
+              collection = overrides[1] || collection
+            end
+          end
+
+          suffix = all_suffix
+
+          # Order of these metrics matters--the first metric in the list will
+          # be treated as the scoped metric in a bunch of different cases.
+          metrics = [
+            operation_metric_for(product, operation),
+            product_suffixed_rollup(product, suffix),
+            product_rollup(product),
+            suffixed_rollup(suffix),
+            ROLLUP_METRIC
+          ]
+
+          metrics.unshift statement_metric_for(product, collection, operation) if collection
+
+          metrics
+        end
+
+        def self.metrics_from_sql(product, sql)
+          operation = Mmtrix::Agent::Database.parse_operation_from_query(sql) || OTHER
+          metrics_for(product, operation)
+        end
+
+        # Allow Transaction#with_database_metric_name to override our
+        # collection and operation
+        def self.overridden_operation_and_collection #THREAD_LOCAL_ACCESS
+          state = Mmtrix::Agent::TransactionState.tl_get
+          txn   = state.current_transaction
+          txn ? txn.instrumentation_state[:datastore_override] : nil
+        end
+
+        # If the override declared a product affiliation, abide by that
+        # ActiveRecord has database-specific product names, so we recognize
+        # it by the generic_product it passes.
+        def self.should_override?(overrides, product, generic_product)
+          override_product = overrides[2]
+
+          override_product.nil? ||
+            override_product == product ||
+            override_product == generic_product
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/datastores/mongo/metric_translator.rb

@@ -0,0 +1,191 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/datastores/mongo/obfuscator'
+require 'mmtrix/agent/datastores/metric_helper'
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        module MetricTranslator
+          def self.metrics_for(name, payload)
+            payload ||= {}
+
+            if collection_in_selector?(payload)
+              command_key = command_key_from_selector(payload)
+              name        = get_name_from_selector(command_key, payload)
+              collection  = get_collection_from_selector(command_key, payload)
+            else
+              collection = payload[:collection]
+            end
+
+            # The 1.10.0 version of the mongo driver renamed 'remove' to
+            # 'delete', but for metric consistency with previous versions we
+            # want to keep it as 'remove'.
+            name = 'remove' if name.to_s == 'delete'
+
+            if self.find_one?(name, payload)
+              name = 'findOne'
+            elsif self.find_and_remove?(name, payload)
+              name = 'findAndRemove'
+            elsif self.find_and_modify?(name, payload)
+              name = 'findAndModify'
+            elsif self.create_indexes?(name, payload)
+              name = 'createIndexes'
+            elsif self.create_index?(name, payload)
+              name = 'createIndex'
+              collection = self.collection_name_from_index(payload)
+            elsif self.drop_indexes?(name, payload)
+              name = 'dropIndexes'
+            elsif self.drop_index?(name, payload)
+              name = 'dropIndex'
+            elsif self.re_index?(name, payload)
+              name = 'reIndex'
+            elsif self.group?(name, payload)
+              name = 'group'
+              collection = collection_name_from_group_selector(payload)
+            elsif self.rename_collection?(name, payload)
+              name = 'renameCollection'
+              collection = collection_name_from_rename_selector(payload)
+            end
+
+            build_metrics(name, collection)
+          rescue => e
+            Mmtrix::Agent.logger.debug("Failure during Mongo metric generation", e)
+            []
+          end
+
+          MONGO_PRODUCT_NAME = "MongoDB".freeze
+
+          def self.build_metrics(name, collection)
+            Mmtrix::Agent::Datastores::MetricHelper.metrics_for(MONGO_PRODUCT_NAME,
+                                                                  name,
+                                                                  collection)
+          end
+
+          def self.collection_in_selector?(payload)
+            payload[:collection] == '$cmd' && payload[:selector]
+          end
+
+          NAMES_IN_SELECTOR = [
+            :findandmodify,
+
+            "aggregate",
+            "count",
+            "group",
+            "mapreduce",
+
+            :distinct,
+
+            :createIndexes,
+            :deleteIndexes,
+            :reIndex,
+
+            :collstats,
+            :renameCollection,
+            :drop,
+          ]
+
+          def self.command_key_from_selector(payload)
+            selector = payload[:selector]
+            NAMES_IN_SELECTOR.find do |check_name|
+              selector.key?(check_name)
+            end
+          end
+
+          def self.get_name_from_selector(command_key, payload)
+            if command_key
+              command_key.to_sym
+            else
+              Mmtrix::Agent.increment_metric("Supportability/Mongo/UnknownCollection")
+              payload[:selector].first.first unless command_key
+            end
+          end
+
+          CMD_COLLECTION = "$cmd".freeze
+
+          def self.get_collection_from_selector(command_key, payload)
+            if command_key
+              payload[:selector][command_key]
+            else
+              Mmtrix::Agent.increment_metric("Supportability/Mongo/UnknownCollection")
+              CMD_COLLECTION
+            end
+          end
+
+          def self.find_one?(name, payload)
+            name == :find && payload[:limit] == -1
+          end
+
+          def self.find_and_modify?(name, payload)
+            name == :findandmodify
+          end
+
+          def self.find_and_remove?(name, payload)
+            name == :findandmodify && payload[:selector] && payload[:selector][:remove]
+          end
+
+          def self.create_indexes?(name, paylod)
+            name == :createIndexes
+          end
+
+          def self.create_index?(name, payload)
+            name == :insert && payload[:collection] == "system.indexes"
+          end
+
+          def self.drop_indexes?(name, payload)
+            name == :deleteIndexes && payload[:selector] && payload[:selector][:index] == "*"
+          end
+
+          def self.drop_index?(name, payload)
+            name == :deleteIndexes
+          end
+
+          def self.re_index?(name, payload)
+            name == :reIndex && payload[:selector] && payload[:selector][:reIndex]
+          end
+
+          def self.group?(name, payload)
+            name == :group
+          end
+
+          def self.rename_collection?(name, payload)
+            name == :renameCollection
+          end
+
+          def self.collection_name_from_index(payload)
+            if payload[:documents]
+              if payload[:documents].is_a?(Array)
+                # mongo gem versions pre 1.10.0
+                document = payload[:documents].first
+              else
+                # mongo gem versions 1.10.0 and later
+                document = payload[:documents]
+              end
+
+              if document && document[:ns]
+                return document[:ns].split('.').last
+              end
+            end
+
+            'system.indexes'
+          end
+
+          def self.collection_name_from_group_selector(payload)
+            payload[:selector]["group"]["ns"]
+          end
+
+          def self.collection_name_from_rename_selector(payload)
+            parts = payload[:selector][:renameCollection].split('.')
+            parts.shift
+            parts.join('.')
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/datastores/mongo/obfuscator.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        module Obfuscator
+
+          WHITELIST = [:operation].freeze
+
+          def self.obfuscate_statement(source, whitelist=WHITELIST)
+            obfuscated = {}
+            source.each do |key, value|
+              if whitelist.include?(key)
+                obfuscated[key] = value
+              else
+                obfuscated[key] = obfuscate_value(value, whitelist)
+              end
+            end
+
+            obfuscated
+          end
+
+          def self.obfuscate_value(value, whitelist)
+            if value.is_a?(Hash)
+              obfuscate_statement(value, whitelist)
+            elsif value.is_a?(Array)
+              value.map {|v| obfuscate_value(v, whitelist)}
+            else
+              '?'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/datastores/mongo/statement_formatter.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/datastores/mongo/obfuscator'
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        module StatementFormatter
+
+          PLAINTEXT_KEYS = [
+            :database,
+            :collection,
+            :operation,
+            :fields,
+            :skip,
+            :limit,
+            :order
+          ]
+
+          OBFUSCATE_KEYS = [
+            :selector
+          ]
+
+          def self.format(statement, operation)
+            return nil unless Mmtrix::Agent.config[:'mongo.capture_queries']
+
+            result = { :operation => operation }
+
+            PLAINTEXT_KEYS.each do |key|
+              result[key] = statement[key] if statement.key?(key)
+            end
+
+            OBFUSCATE_KEYS.each do |key|
+              if statement.key?(key) && statement[key]
+                obfuscated = obfuscate(statement[key])
+                result[key] = obfuscated if obfuscated
+              end
+            end
+            result
+          end
+
+          def self.obfuscate(statement)
+            statement = Obfuscator.obfuscate_statement(statement) if Mmtrix::Agent.config[:'mongo.obfuscate_queries']
+            statement
+          end
+        end
+      end
+    end
+  end
+end