newrelic_rpm 3.12.0.288 → 9.10.2

data/lib/new_relic/agent/database.rb

@@ -1,40 +1,39 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 require 'singleton'
-require 'new_relic/agent/database/obfuscation_helpers'
+require 'new_relic/agent/database/explain_plan_helpers'
 require 'new_relic/agent/database/obfuscator'
-require 'new_relic/agent/database/postgres_explain_obfuscator'
 
 module NewRelic
-  # columns for a mysql explain plan
-  MYSQL_EXPLAIN_COLUMNS = [
-                           "Id",
-                           "Select Type",
-                           "Table",
-                           "Type",
-                           "Possible Keys",
-                           "Key",
-                           "Key Length",
-                           "Ref",
-                           "Rows",
-                           "Extra"
-                          ].freeze
-
   module Agent
     module Database
       MAX_QUERY_LENGTH = 16384
+      ELLIPSIS = '...'.freeze
 
       extend self
 
+      # Properly encode, truncate, and dup the incoming query.
+      # Take care not to the dup the query more than once as
+      # correctly encoded may also dup the query.
       def capture_query(query)
-        Helper.correctly_encoded(truncate_query(query))
+        return unless query
+
+        id = query.object_id
+        query = Helper.correctly_encoded(truncate_query(query))
+        if query.object_id == id
+          query.dup
+        else
+          query
+        end
       end
 
       def truncate_query(query)
+        return unless query
+
         if query.length > (MAX_QUERY_LENGTH - 4)
-          query[0..MAX_QUERY_LENGTH - 4] + '...'
+          query[0..MAX_QUERY_LENGTH - 4] << ELLIPSIS
         else
           query
         end
@@ -48,7 +47,7 @@ module NewRelic
         Obfuscator.instance.set_sql_obfuscator(type, &block)
       end
 
-      def record_sql_method(config_section=:transaction_tracer)
+      def record_sql_method(config_section = :transaction_tracer)
         key = record_sql_method_key(config_section)
 
         case Agent.config[key].to_s
@@ -78,11 +77,11 @@ module NewRelic
 
       RECORD_FOR = [:raw, :obfuscated].freeze
 
-      def should_record_sql?(config_section=:transaction_tracer)
+      def should_record_sql?(config_section = :transaction_tracer)
         RECORD_FOR.include?(record_sql_method(config_section))
       end
 
-      def should_collect_explain_plans?(config_section=:transaction_tracer)
+      def should_collect_explain_plans?(config_section = :transaction_tracer)
         should_record_sql?(config_section) &&
           Agent.config["#{config_section}.explain_enabled".to_sym]
       end
@@ -95,190 +94,43 @@ module NewRelic
         ConnectionManager.instance.close_connections
       end
 
-      # This takes a connection config hash from ActiveRecord or Sequel and
-      # returns a string describing the associated database adapter
-      def adapter_from_config(config)
-        if config[:adapter]
-          return config[:adapter].to_s
-        elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
-          # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
-          # jdbc-sqlite3 gems.
-          return $1
-        end
-      end
-
       # Perform this in the runtime environment of a managed
       # application, to explain the sql statement executed within a
-      # node of a transaction sample. Returns an array of
-      # explanations (which is an array rows consisting of an array of
-      # strings for each column returned by the the explain query)
-      # Note this happens only for statements whose execution time
-      # exceeds a threshold (e.g. 500ms) and only within the slowest
-      # transaction in a report period, selected for shipment to New
-      # Relic
-      def explain_sql(sql, connection_config, &explainer)
-        return nil unless sql && connection_config
-        statement = sql.split(";\n")[0] # only explain the first
-        explain_plan = explain_statement(statement, connection_config, &explainer)
-        return explain_plan || []
-      end
-
-      SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
-
-      def explain_statement(statement, config, &explainer)
-        return unless is_select?(statement)
-
-        if statement[-3,3] == '...'
-          NewRelic::Agent.logger.debug('Unable to collect explain plan for truncated query.')
-          return
-        end
-
-        if parameterized?(statement)
-          NewRelic::Agent.logger.debug('Unable to collect explain plan for parameterized query.')
-          return
-        end
-
-        adapter = adapter_from_config(config)
-        if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
-          NewRelic::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
-          return
-        end
-
-        handle_exception_in_explain do
-          start = Time.now
-          plan = explainer.call(config, statement)
-          ::NewRelic::Agent.record_metric("Supportability/Database/execute_explain_plan", Time.now - start)
-          return process_resultset(plan, adapter) if plan
-        end
+      # node of a transaction sample. Returns an array of two arrays.
+      # The first array contains the headers, while the second consists of
+      # arrays of strings for each column returned by the explain query.
+      # Note this happens only for statements whose execution time exceeds
+      # a threshold (e.g. 500ms) and only within the slowest transaction
+      # in a report period, selected for shipment to New Relic
+      def explain_sql(statement)
+        return nil unless statement.sql && statement.explainer && statement.config
+
+        statement.sql = statement.sql.split(";\n")[0] # only explain the first
+        return statement.explain || []
       end
 
-      def process_resultset(results, adapter)
-        case adapter.to_s
-        when 'postgres', 'postgresql'
-          process_explain_results_postgres(results)
-        when 'mysql2'
-          process_explain_results_mysql2(results)
-        when 'mysql'
-          process_explain_results_mysql(results)
-        when 'sqlite'
-          process_explain_results_sqlite(results)
-        end
-      end
-
-      QUERY_PLAN = 'QUERY PLAN'.freeze
-
-      def process_explain_results_postgres(results)
-        if results.is_a?(String)
-          query_plan_string = results
-        else
-          lines = []
-          results.each { |row| lines << row[QUERY_PLAN] }
-          query_plan_string = lines.join("\n")
-        end
-
-        unless record_sql_method == :raw
-          query_plan_string = NewRelic::Agent::Database::PostgresExplainObfuscator.obfuscate(query_plan_string)
-        end
-        values = query_plan_string.split("\n").map { |line| [line] }
-
-        [[QUERY_PLAN], values]
-      end
-
-      # Sequel returns explain plans as just one big pre-formatted String
-      # In that case, we send a nil headers array, and the single string
-      # wrapped in an array for the values.
-      # Note that we don't use this method for Postgres explain plans, since
-      # they need to be passed through the explain plan obfuscator first.
-      def string_explain_plan_results(results)
-        [nil, [results]]
-      end
-
-      def process_explain_results_mysql(results)
-        return string_explain_plan_results(results) if results.is_a?(String)
-        headers = []
-        values  = []
-        if results.is_a?(Array)
-          # We're probably using the jdbc-mysql gem for JRuby, which will give
-          # us an array of hashes.
-          headers = results.first.keys
-          results.each do |row|
-            values << headers.map { |h| row[h] }
-          end
-        else
-          # We're probably using the native mysql driver gem, which will give us
-          # a Mysql::Result object that responds to each_hash
-          results.each_hash do |row|
-            headers = row.keys
-            values << headers.map { |h| row[h] }
-          end
-        end
-        [headers, values]
-      end
-
-      def process_explain_results_mysql2(results)
-        return string_explain_plan_results(results) if results.is_a?(String)
-        headers = results.fields
-        values  = []
-        results.each { |row| values << row }
-        [headers, values]
-      end
-
-      SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
-
-      def process_explain_results_sqlite(results)
-        return string_explain_plan_results(results) if results.is_a?(String)
-        headers = SQLITE_EXPLAIN_COLUMNS
-        values  = []
-        results.each do |row|
-          values << headers.map { |h| row[h] }
-        end
-        [headers, values]
-      end
-
-      def handle_exception_in_explain
-        yield
-      rescue => e
-        begin
-          # guarantees no throw from explain_sql
-          ::NewRelic::Agent.logger.error("Error getting query plan:", e)
-          nil
-        rescue
-          # double exception. throw up your hands
-          nil
-        end
-      end
-
-      KNOWN_OPERATIONS = [
-        'alter',
-        'select',
-        'update',
-        'delete',
-        'insert',
-        'create',
-        'show',
-        'set',
-        'exec',
-        'execute',
-        'call'
+      KNOWN_OPERATIONS = %w[
+        alter
+        select
+        update
+        delete
+        insert
+        create
+        show
+        set
+        exec
+        execute
+        call
       ]
-
+      OTHER_OPERATION = 'other'.freeze
       SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
-      EMPTY_STRING      = ''.freeze
 
       def parse_operation_from_query(sql)
-        sql = sql.gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
-        if sql =~ /(\w+)/
-          op = $1.downcase
-          return op if KNOWN_OPERATIONS.include?(op)
-        end
-      end
+        sql = Helper.correctly_encoded(sql).gsub(SQL_COMMENT_REGEX, NewRelic::EMPTY_STR)
+        return unless sql =~ /(\w+)/
 
-      def is_select?(statement)
-        parse_operation_from_query(statement) == 'select'
-      end
-
-      def parameterized?(statement)
-        Obfuscator.instance.obfuscate_single_quote_literals(statement) =~ /\$\d+/
+        op = Regexp.last_match(1).downcase
+        KNOWN_OPERATIONS.include?(op) ? op : OTHER_OPERATION
       end
 
       class ConnectionManager
@@ -296,9 +148,9 @@ module NewRelic
           return connection if connection
 
           begin
-            @connections[config] = connector.call(config)
+            @connections[config] = yield(config)
           rescue => e
-            ::NewRelic::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
+            ::NewRelic::Agent.logger.error('Caught exception trying to get connection to DB for explain.', e)
             nil
           end
         end
@@ -317,8 +169,120 @@ module NewRelic
         end
       end
 
-      class Statement < String
-        attr_accessor :adapter, :config, :explainer
+      class Statement
+        include ExplainPlanHelpers
+
+        attr_accessor :sql, :config, :explainer, :binds, :name, :host, :port_path_or_id, :database_name
+
+        DEFAULT_QUERY_NAME = 'SQL'.freeze
+
+        def initialize(sql, config = {}, explainer = nil, binds = nil, name = DEFAULT_QUERY_NAME, host = nil, port_path_or_id = nil, database_name = nil)
+          @sql = Database.capture_query(sql)
+          @config = config
+          @explainer = explainer
+          @binds = binds
+          @name = name
+          @host = host
+          @port_path_or_id = port_path_or_id
+          @database_name = database_name
+          @safe_sql = nil
+        end
+
+        # Returns an sql statement that will be in the form most permissable by
+        # the config. The format will be safe for transmission to New Relic.
+        def safe_sql
+          @safe_sql ||= case Database.record_sql_method
+            when :obfuscated
+              Database.obfuscate_sql(self)
+            when :raw
+              sql.to_s
+          end
+        end
+
+        # This takes a connection config hash from ActiveRecord or Sequel and
+        # returns a symbol describing the associated database adapter
+        def adapter
+          return unless @config
+
+          @adapter ||= if @config[:adapter]
+            symbolized_adapter(@config[:adapter].to_s.downcase)
+          elsif @config[:uri] && @config[:uri].to_s =~ /^jdbc:([^:]+):/
+            # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or jdbc-sqlite3 gems.
+            symbolized_adapter($1)
+          end
+        end
+
+        def explain
+          return unless explainable?
+
+          handle_exception_in_explain do
+            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            plan = @explainer.call(self)
+            ::NewRelic::Agent.record_metric(
+              'Supportability/Database/execute_explain_plan',
+              Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+            )
+            return process_resultset(plan, adapter) if plan
+          end
+        end
+
+        NEWLINE = "\n".freeze
+
+        def append_sql(new_sql)
+          return if new_sql.empty?
+
+          @sql = Database.truncate_query(@sql << NEWLINE << new_sql)
+        end
+
+        private
+
+        POSTGIS_PREFIX = 'postgis'.freeze
+        POSTGRES_PREFIX = 'postgres'.freeze
+        MYSQL_PREFIX = 'mysql'.freeze
+        MYSQL2_PREFIX = 'mysql2'.freeze
+        SQLITE_PREFIX = 'sqlite'.freeze
+
+        def symbolized_adapter(adapter)
+          if adapter.start_with?(POSTGRES_PREFIX) || adapter == POSTGIS_PREFIX
+            :postgres
+          elsif adapter == MYSQL_PREFIX
+            :mysql
+          # For the purpose of fetching explain plans, we need to maintain the distinction
+          # between usage of mysql and mysql2. Obfuscation is the same, though.
+          elsif adapter == MYSQL2_PREFIX
+            :mysql2
+          elsif adapter.start_with?(SQLITE_PREFIX)
+            :sqlite
+          else
+            adapter.to_sym
+          end
+        end
+
+        def explainable?
+          return false unless @explainer && is_select?(@sql)
+
+          if @sql.end_with?(ELLIPSIS)
+            NewRelic::Agent.logger.debug('Unable to collect explain plan for truncated query.')
+            return false
+          end
+
+          if parameterized?(@sql) && (!@binds || @binds.empty?)
+            NewRelic::Agent.logger.debug('Unable to collect explain plan for parameter-less parameterized query.')
+            return false
+          end
+
+          if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
+            NewRelic::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
+            return false
+          end
+
+          if multiple_queries?(@sql)
+            NewRelic::Agent.logger.debug('Unable to collect explain plan for multiple queries.')
+            return false
+          end
+
+          true
+        end
       end
     end
   end

data/lib/new_relic/agent/database_adapter.rb

@@ -0,0 +1,35 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic
+  module Agent
+    class DatabaseAdapter
+      VERSIONS = {
+        '6.1.0' => proc { ActiveRecord::Base.connection_db_config.configuration_hash[:adapter] },
+        '4.0.0' => proc { ActiveRecord::Base.connection_config[:adapter] },
+        '3.0.0' => proc { |env| ActiveRecord::Base.configurations[env]['adapter'] }
+      }
+
+      def self.value
+        return unless defined? ActiveRecord::Base
+
+        new(::NewRelic::Control.instance.env, ActiveRecord::VERSION::STRING).value
+      end
+
+      attr_reader :env, :version
+
+      def initialize(env, version)
+        @env = env
+        @version = Gem::Version.new(version)
+      end
+
+      def value
+        match = VERSIONS.keys.find { |key| version >= Gem::Version.new(key) }
+        return unless match
+
+        VERSIONS[match].call(env)
+      end
+    end
+  end
+end

data/lib/new_relic/agent/datastores/metric_helper.rb

@@ -1,20 +1,17 @@
-# encoding: utf-8
 # This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
 
 module NewRelic
   module Agent
     module Datastores
       module MetricHelper
-        ROLLUP_METRIC        = "Datastore/all".freeze
-        WEB_ROLLUP_METRIC    = "Datastore/allWeb".freeze
-        OTHER_ROLLUP_METRIC  = "Datastore/allOther".freeze
-        DEFAULT_PRODUCT_NAME = "ActiveRecord".freeze
-        OTHER = "Other".freeze
+        ROLLUP_METRIC = 'Datastore/all'.freeze
+        OTHER = 'Other'.freeze
 
-        ALL = "all".freeze
-        ALL_WEB = "allWeb".freeze
-        ALL_OTHER = "allOther".freeze
+        ALL = 'all'.freeze
+        ALL_WEB = 'allWeb'.freeze
+        ALL_OTHER = 'allOther'.freeze
 
         def self.statement_metric_for(product, collection, operation)
           "Datastore/statement/#{product}/#{collection}/#{operation}"
@@ -24,6 +21,10 @@ module NewRelic
           "Datastore/operation/#{product}/#{operation}"
         end
 
+        def self.instance_metric_for(product, host, port_path_or_id)
+          "Datastore/instance/#{product}/#{host}/#{port_path_or_id}"
+        end
+
         def self.product_suffixed_rollup(product, suffix)
           "Datastore/#{product}/#{suffix}"
         end
@@ -44,42 +45,81 @@ module NewRelic
           end
         end
 
-        def self.metrics_for(product, operation, collection = nil)
-          if overrides = overridden_operation_and_collection
-            operation  = overrides[0] || operation
-            collection = overrides[1] || collection
+        def self.scoped_metric_for(product, operation, collection = nil)
+          if collection
+            statement_metric_for(product, collection, operation)
+          else
+            operation_metric_for(product, operation)
           end
+        end
 
+        def self.unscoped_metrics_for(product, operation, collection = nil, host = nil, port_path_or_id = nil)
           suffix = all_suffix
 
-          # Order of these metrics matters--the first metric in the list will
-          # be treated as the scoped metric in a bunch of different cases.
           metrics = [
-            operation_metric_for(product, operation),
             product_suffixed_rollup(product, suffix),
             product_rollup(product),
             suffixed_rollup(suffix),
             ROLLUP_METRIC
           ]
 
-          metrics.unshift statement_metric_for(product, collection, operation) if collection
+          if NewRelic::Agent.config[:'datastore_tracer.instance_reporting.enabled'] && host && port_path_or_id
+            metrics.unshift(instance_metric_for(product, host, port_path_or_id))
+          end
+          metrics.unshift(operation_metric_for(product, operation)) if collection
+
+          metrics
+        end
+
+        def self.product_operation_collection_for(product, operation, collection = nil, generic_product = nil)
+          if overrides = overridden_operation_and_collection
+            if should_override?(overrides, product, generic_product)
+              operation = overrides[0] || operation
+              collection = overrides[1] || collection
+            end
+          end
+          [product, operation, collection]
+        end
+
+        def self.metrics_for(product, operation, collection = nil, generic_product = nil, host = nil, port_path_or_id = nil)
+          product, operation, collection = product_operation_collection_for(product, operation, collection, generic_product)
+
+          # Order of these metrics matters--the first metric in the list will
+          # be treated as the scoped metric in a bunch of different cases.
+          metrics = unscoped_metrics_for(product, operation, collection, host, port_path_or_id)
+          metrics.unshift(scoped_metric_for(product, operation, collection))
 
           metrics
         end
 
         def self.metrics_from_sql(product, sql)
-          operation = NewRelic::Agent::Database.parse_operation_from_query(sql) || OTHER
+          operation = operation_from_sql(sql)
           metrics_for(product, operation)
         end
 
+        def self.operation_from_sql(sql)
+          operation = NewRelic::Agent::Database.parse_operation_from_query(sql)
+          operation = OTHER if operation.eql?(NewRelic::Agent::Database::OTHER_OPERATION)
+          operation
+        end
+
         # Allow Transaction#with_database_metric_name to override our
         # collection and operation
-        def self.overridden_operation_and_collection #THREAD_LOCAL_ACCESS
-          state = NewRelic::Agent::TransactionState.tl_get
-          txn   = state.current_transaction
+        def self.overridden_operation_and_collection # THREAD_LOCAL_ACCESS
+          txn = Tracer.current_transaction
           txn ? txn.instrumentation_state[:datastore_override] : nil
         end
 
+        # If the override declared a product affiliation, abide by that
+        # ActiveRecord has database-specific product names, so we recognize
+        # it by the generic_product it passes.
+        def self.should_override?(overrides, product, generic_product)
+          override_product = overrides[2]
+
+          override_product.nil? ||
+            override_product == product ||
+            override_product == generic_product
+        end
       end
     end
   end

data/lib/new_relic/agent/datastores/mongo/event_formatter.rb

@@ -0,0 +1,50 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require_relative '../nosql_obfuscator'
+
+module NewRelic
+  module Agent
+    module Datastores
+      module Mongo
+        module EventFormatter
+          # Keys that will get their values replaced with '?'.
+          OBFUSCATE_KEYS = %w[filter query pipeline].freeze
+
+          # Keys that will get completely removed from the statement.
+          DENYLISTED_KEYS = %w[deletes documents updates].freeze
+
+          def self.format(command_name, database_name, command)
+            return nil unless NewRelic::Agent.config[:'mongo.capture_queries']
+
+            result = {
+              :operation => command_name,
+              :database => database_name,
+              :collection => command.values.first
+            }
+
+            command.each do |key, value|
+              next if DENYLISTED_KEYS.include?(key)
+
+              if OBFUSCATE_KEYS.include?(key)
+                obfuscated = obfuscate(value)
+                result[key] = obfuscated if obfuscated
+              else
+                result[key] = value
+              end
+            end
+            result
+          end
+
+          def self.obfuscate(statement)
+            if NewRelic::Agent.config[:'mongo.obfuscate_queries']
+              statement = NewRelic::Agent::Datastores::NosqlObfuscator.obfuscate_statement(statement)
+            end
+            statement
+          end
+        end
+      end
+    end
+  end
+end