newrelic_rpm 3.14.2.312 → 3.14.3.313

data/lib/new_relic/agent/database.rb

@@ -327,7 +327,23 @@ module NewRelic
         end
 
         def adapter
-          config && config[:adapter]
+          config && config[:adapter] && symbolized_adapter(config[:adapter].to_s.downcase)
+        end
+
+        POSTGRES_PREFIX = 'postgres'.freeze
+        MYSQL_PREFIX = 'mysql'.freeze
+        SQLITE_PREFIX = 'sqlite'.freeze
+
+        def symbolized_adapter(adapter)
+          if adapter.start_with? POSTGRES_PREFIX
+            :postgres
+          elsif adapter.start_with? MYSQL_PREFIX
+            :mysql
+          elsif adapter.start_with? SQLITE_PREFIX
+            :sqlite
+          else
+            adapter.to_sym
+          end
         end
       end
     end

data/lib/new_relic/agent/database/obfuscation_helpers.rb

@@ -6,71 +6,91 @@ module NewRelic
   module Agent
     module Database
       module ObfuscationHelpers
-        # Note that the following two regexes are applied to a reversed version
-        # of the query. This is why the backslash escape sequences (\' and \")
-        # appear reversed within them.
-        #
-        # Note that some database adapters (notably, PostgreSQL with
-        # standard_conforming_strings on and MySQL with NO_BACKSLASH_ESCAPES on)
-        # do not apply special treatment to backslashes within quoted string
-        # literals. We don't have an easy way of determining whether the
-        # database connection from which a query was captured was operating in
-        # one of these modes, but the obfuscation is done in such a way that it
-        # should not matter.
-        #
-        # Reversing the query string before obfuscation allows us to get around
-        # the fact that a \' appearing within a string may or may not terminate
-        # the string, because we know that a string cannot *start* with a \'.
-        REVERSE_SINGLE_QUOTES_REGEX = /'(?:''|'\\|[^'])*'/
-        REVERSE_ANY_QUOTES_REGEX    = /'(?:''|'\\|[^'])*'|"(?:""|"\\|[^"])*"/
+        COMPONENTS_REGEX_MAP = {
+          :single_quotes => /'(?:[^']|'')*?(?:\\'.*|'(?!'))/,
+          :double_quotes => /"(?:[^"]|"")*?(?:\\".*|"(?!"))/,
+          :dollar_quotes => /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/,
+          :uuids => /\{?(?:[0-9a-fA-F]\-*){32}\}?/,
+          :numeric_literals => /\b-?(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/,
+          :boolean_literals => /\b(?:true|false|null)\b/i,
+          :hexadecimal_literals => /0x[0-9a-fA-F]+/,
+          :comments => /(?:#|--).*?(?=\r|\n|$)/i,
+          :multi_line_comments => /\/\*(?:[^\/]|\/[^*])*?(?:\*\/|\/\*.*)/,
+          :oracle_quoted_strings => /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/
+        }
 
-        NUMERICS_REGEX = /\b\d+\b/
-
-        # We take a conservative, overly-aggressive approach to obfuscating
-        # comments, and drop everything from the query after encountering any
-        # character sequence that could be a comment initiator. We do this after
-        # removal of string literals to avoid accidentally over-obfuscating when
-        # a string literal contains a comment initiator.
-        SQL_COMMENT_REGEX = Regexp.new('(?:/\*|--|#).*', Regexp::MULTILINE).freeze
+        DIALECT_COMPONENTS = {
+          :fallback   => COMPONENTS_REGEX_MAP.keys,
+          :mysql      => [:single_quotes, :double_quotes, :numeric_literals, :boolean_literals,
+                          :hexadecimal_literals, :comments, :multi_line_comments],
+          :postgres   => [:single_quotes, :dollar_quotes, :uuids, :numeric_literals,
+                          :boolean_literals, :comments, :multi_line_comments],
+          :sqlite     => [:single_quotes, :numeric_literals, :boolean_literals, :hexadecimal_literals,
+                          :comments, :multi_line_comments],
+          :oracle     => [:single_quotes, :oracle_quoted_strings, :numeric_literals, :comments,
+                          :multi_line_comments],
+          :cassandra  => [:single_quotes, :uuids, :numeric_literals, :boolean_literals,
+                          :hexadecimal_literals, :comments, :multi_line_comments]
+        }
 
         # We use these to check whether the query contains any quote characters
         # after obfuscation. If so, that's a good indication that the original
-        # query was malformed, and so our obfuscation can't reliabily find
+        # query was malformed, and so our obfuscation can't reliably find
         # literals. In such a case, we'll replace the entire query with a
         # placeholder.
-        LITERAL_SINGLE_QUOTE = "'".freeze
-        LITERAL_DOUBLE_QUOTE = '"'.freeze
+        CLEANUP_REGEX = {
+          :mysql => /'|"|\/\*|\*\//,
+          :postgres => /'|\/\*|\*\/|\$(?!\?)/,
+          :sqlite => /'|\/\*|\*\/|\$/,
+          :cassandra => /'|\/\*|\*\//,
+          :oracle => /'|\/\*|\*\//
+        }
 
         PLACEHOLDER = '?'.freeze
+        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation".freeze
 
         def obfuscate_single_quote_literals(sql)
-          obfuscated = sql.reverse
-          obfuscated.gsub!(REVERSE_SINGLE_QUOTES_REGEX, PLACEHOLDER)
-          obfuscated.reverse!
-          obfuscated
+          sql.gsub!(COMPONENTS_REGEX_MAP[:single_quotes], PLACEHOLDER) || sql
         end
 
-        def obfuscate_quoted_literals(sql)
-          obfuscated = sql.reverse
-          obfuscated.gsub!(REVERSE_ANY_QUOTES_REGEX, PLACEHOLDER)
-          obfuscated.reverse!
-          obfuscated
+        def self.generate_regex(dialect)
+          components = DIALECT_COMPONENTS[dialect]
+          Regexp.union(components.map{|component| COMPONENTS_REGEX_MAP[component]})
         end
 
-        def obfuscate_numeric_literals(sql)
-          sql.gsub(NUMERICS_REGEX, PLACEHOLDER)
-        end
+        MYSQL_COMPONENTS_REGEX = self.generate_regex(:mysql)
+        POSTGRES_COMPONENTS_REGEX = self.generate_regex(:postgres)
+        SQLITE_COMPONENTS_REGEX = self.generate_regex(:sqlite)
+        ORACLE_COMPONENTS_REGEX = self.generate_regex(:oracle)
+        CASSANDRA_COMPONENTS_REGEX = self.generate_regex(:cassandra)
+        FALLBACK_REGEX = self.generate_regex(:fallback)
 
-        def remove_comments(sql)
-          sql.gsub(SQL_COMMENT_REGEX, PLACEHOLDER)
-        end
-
-        def contains_single_quotes?(str)
-          str.include?(LITERAL_SINGLE_QUOTE)
+        def obfuscate(sql, adapter)
+          case adapter
+          when :mysql
+            regex = MYSQL_COMPONENTS_REGEX
+          when :postgres
+            regex = POSTGRES_COMPONENTS_REGEX
+          when :sqlite
+            regex = SQLITE_COMPONENTS_REGEX
+          when :oracle
+            regex = ORACLE_COMPONENTS_REGEX
+          when :cassandra
+            regex = CASSANDRA_COMPONENTS_REGEX
+          else
+            regex = FALLBACK_REGEX
+          end
+          obfuscated = sql.gsub(regex, PLACEHOLDER)
+          obfuscated = FAILED_TO_OBFUSCATE_MESSAGE if detect_unmatched_pairs(obfuscated, adapter)
+          obfuscated
         end
 
-        def contains_quotes?(str)
-          str.include?(LITERAL_SINGLE_QUOTE) || str.include?(LITERAL_DOUBLE_QUOTE)
+        def detect_unmatched_pairs(obfuscated, adapter)
+          if CLEANUP_REGEX[adapter]
+            CLEANUP_REGEX[adapter].match(obfuscated)
+          else
+            CLEANUP_REGEX[:mysql].match(obfuscated)
+          end
         end
       end
     end

data/lib/new_relic/agent/database/obfuscator.rb

@@ -13,8 +13,8 @@ module NewRelic
 
         attr_reader :obfuscator
 
-        QUERY_TOO_LARGE_MESSAGE     = "Query too large (over 16k characters) to safely obfuscate"
-        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
+        QUERY_TOO_LARGE_MESSAGE = "Query too large (over 16k characters) to safely obfuscate".freeze
+        ELLIPSIS = "...".freeze
 
         def initialize
           reset
@@ -50,30 +50,11 @@ module NewRelic
         def default_sql_obfuscator(sql)
           stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
 
-          if stmt.sql[-3,3] == '...'
+          if stmt.sql.end_with? ELLIPSIS
             return QUERY_TOO_LARGE_MESSAGE
           end
 
-          obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
-
-          obfuscated = obfuscate_numeric_literals(stmt.sql)
-
-          if obfuscate_double_quotes
-            obfuscated = obfuscate_quoted_literals(obfuscated)
-            obfuscated = remove_comments(obfuscated)
-            if contains_quotes?(obfuscated)
-              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
-            end
-          else
-            obfuscated = obfuscate_single_quote_literals(obfuscated)
-            obfuscated = remove_comments(obfuscated)
-            if contains_single_quotes?(obfuscated)
-              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
-            end
-          end
-
-
-          obfuscated.to_s # return back to a regular String
+          obfuscate(stmt.sql, stmt.adapter).to_s
         end
       end
     end

data/lib/new_relic/agent/database/postgres_explain_obfuscator.rb

@@ -12,7 +12,7 @@ module NewRelic
 
         # Note that this regex can't be shared with the ones in the
         # Database::Obfuscator class because here we don't look for
-        # backslash-escaped strings (and those regexes are backwards).
+        # backslash-escaped strings.
         QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
         LABEL_LINE_REGEX     = /^([^:\n]*:\s+).*$/.freeze
 

data/lib/new_relic/agent/instrumentation/data_mapper.rb

@@ -109,6 +109,9 @@ module NewRelic
       end
 
       DATA_MAPPER = "DataMapper".freeze
+      PASSWORD_REGEX = /&password=.*?&/
+      AMPERSAND = '&'.freeze
+      PASSWORD_PARAM = '&password='.freeze
 
       def self.method_body(clazz, method_name, operation_only)
         use_model_name   = NewRelic::Helper.instance_methods_include?(clazz, :model)
@@ -136,7 +139,23 @@ module NewRelic
               name)
 
             NewRelic::Agent::MethodTracer.trace_execution_scoped(metrics) do
-              self.send("#{method_name}_without_newrelic", *args, &blk)
+              begin
+                self.send("#{method_name}_without_newrelic", *args, &blk)
+              rescue ::DataObjects::SQLError => e
+                e.uri.gsub!(PASSWORD_REGEX, AMPERSAND) if e.uri.include?(PASSWORD_PARAM)
+
+                strategy = NewRelic::Agent::Database.record_sql_method(:slow_sql)
+                case strategy
+                when :obfuscated
+                  statement = NewRelic::Agent::Database::Statement.new(e.query, :adapter => self.options[:adapter])
+                  obfuscated_sql = NewRelic::Agent::Database.obfuscate_sql(statement)
+                  e.instance_variable_set(:@query, obfuscated_sql)
+                when :off
+                  e.instance_variable_set(:@query, nil)
+                end
+
+                raise
+              end
             end
           end
         end

data/lib/new_relic/agent/instrumentation/evented_subscriber.rb

@@ -64,7 +64,7 @@ module NewRelic
       # with a couple minor additions so we don't have a hard
       # dependency on ActiveSupport::Notifications.
       #
-      # Represents an intrumentation event, provides timing and metric
+      # Represents an instrumentation event, provides timing and metric
       # name information useful when recording metrics.
       class Event
         attr_reader :name, :time, :transaction_id, :payload, :children

data/lib/new_relic/agent/rules_engine.rb

@@ -4,6 +4,7 @@
 
 require 'new_relic/agent/rules_engine/replacement_rule'
 require 'new_relic/agent/rules_engine/segment_terms_rule'
+require 'new_relic/language_support'
 
 module NewRelic
   module Agent
@@ -26,12 +27,48 @@ module NewRelic
         txn_name_specs     = connect_response['transaction_name_rules']    || []
         segment_rule_specs = connect_response['transaction_segment_terms'] || []
 
-        txn_name_rules = txn_name_specs.map     { |s| ReplacementRule.new(s) }
-        segment_rules  = segment_rule_specs.map { |s| SegmentTermsRule.new(s) }
+        txn_name_rules = txn_name_specs.map { |s| ReplacementRule.new(s) }
+
+        segment_rules = []
+
+        segment_rule_specs.each do |spec|
+          if spec[SegmentTermsRule::PREFIX_KEY] && SegmentTermsRule.valid?(spec)
+            # Build segment_rules in reverse order from which they're provided,
+            # so that when we eliminate duplicates with #uniq!, we retain the last
+            # instances of repeated rules.
+            segment_rules.unshift SegmentTermsRule.new(spec)
+          end
+        end
+
+        reject_rules_with_duplicate_prefixes!(segment_rules)
+
+        segment_rules.reverse! # Reset the rules to their original order.
 
         self.new(txn_name_rules, segment_rules)
       end
 
+      # When multiple rules share the same prefix,
+      # only apply the rule with the last instance of the prefix.
+      # Note that the incoming rules are in reverse order to facilitate this.
+      if NewRelic::LanguageSupport.uniq_accepts_block?
+        def self.reject_rules_with_duplicate_prefixes!(rules)
+          rules.uniq! { |rule| rule.prefix }
+        end
+      else
+        def self.reject_rules_with_duplicate_prefixes!(rules)
+          unique_rules = {}
+
+          rules.reject! do |rule|
+            if unique_rules[rule.prefix]
+              true
+            else
+              unique_rules[rule.prefix] = rule
+              false
+            end
+          end
+        end
+      end
+
       def initialize(rules=[], segment_term_rules=[])
         @rules = rules.sort
         @segment_term_rules = segment_term_rules

data/lib/new_relic/agent/rules_engine/segment_terms_rule.rb

@@ -6,15 +6,32 @@ module NewRelic
   module Agent
     class RulesEngine
       class SegmentTermsRule
+        PREFIX_KEY                        = 'prefix'.freeze
+        TERMS_KEY                         = 'terms'.freeze
         SEGMENT_PLACEHOLDER               = '*'.freeze
         ADJACENT_PLACEHOLDERS_REGEX       = %r{((?:^|/)\*)(?:/\*)*}.freeze
         ADJACENT_PLACEHOLDERS_REPLACEMENT = '\1'.freeze
+        VALID_PREFIX_SEGMENT_COUNT        = 2
 
         attr_reader :prefix, :terms
 
+        def self.valid?(rule_spec)
+          rule_spec[PREFIX_KEY].kind_of?(String) &&
+          rule_spec[TERMS_KEY].kind_of?(Array) &&
+          valid_prefix_segment_count?(rule_spec[PREFIX_KEY])
+        end
+
+        def self.valid_prefix_segment_count?(prefix)
+          count = prefix.count(SEGMENT_SEPARATOR)
+          rindex = prefix.rindex(SEGMENT_SEPARATOR)
+
+          (count == 2 && prefix[rindex + 1].nil?) ||
+          (count == 1 && !prefix[rindex + 1].nil?)
+        end
+
         def initialize(options)
-          @prefix          = options['prefix']
-          @terms           = options['terms']
+          @prefix          = options[PREFIX_KEY]
+          @terms           = options[TERMS_KEY]
           @trim_range      = (@prefix.size..-1)
         end
 
@@ -23,14 +40,19 @@ module NewRelic
         end
 
         def matches?(string)
-          string.start_with?(@prefix)
+          string.start_with?(@prefix) &&
+            (prefix_matches_on_segment_boundary?(string) || string.size == @prefix.size)
+        end
+
+        def prefix_matches_on_segment_boundary?(string)
+          string.size > @prefix.size &&
+            string[@prefix.chomp(SEGMENT_SEPARATOR).size].chr == SEGMENT_SEPARATOR
         end
 
         def apply(string)
           rest          = string[@trim_range]
           leading_slash = rest.slice!(LEADING_SLASH_REGEX)
-
-          segments = rest.split(SEGMENT_SEPARATOR)
+          segments = rest.split(SEGMENT_SEPARATOR, -1)
           segments.map! { |s| @terms.include?(s) ? s : SEGMENT_PLACEHOLDER }
           transformed_suffix = collapse_adjacent_placeholder_segments(segments)
 

data/lib/new_relic/agent/sql_sampler.rb

@@ -313,10 +313,14 @@ module NewRelic
 
       private
 
+      # need to hash the same way in every process, to be able to aggregate slow SQL traces
       def consistent_hash(string)
-        # need to hash the same way in every process
-        Digest::MD5.hexdigest(string).hex \
-          .modulo(2**31-1)  # ensure sql_id fits in an INT(11)
+        if NewRelic::Agent.config[:'slow_sql.use_longer_sql_id']
+          Digest::MD5.hexdigest(string).hex.modulo(2**63-1)
+        else
+          # from when sql_id needed to fit in an INT(11)
+          Digest::MD5.hexdigest(string).hex.modulo(2**31-1)
+        end
       end
     end
   end

data/lib/new_relic/language_support.rb

@@ -5,6 +5,8 @@
 module NewRelic::LanguageSupport
   extend self
 
+  RUBY_VERSION_192 = '1.9.2'.freeze
+
   # need to use syck rather than psych when possible
   def needs_syck?
     !NewRelic::LanguageSupport.using_engine?('jruby') &&
@@ -90,6 +92,12 @@ module NewRelic::LanguageSupport
     end
   end
 
+  if ::RUBY_VERSION >= RUBY_VERSION_192
+    def uniq_accepts_block?; true; end
+  else
+    def uniq_accepts_block?; false; end
+  end
+
   def jruby?
     defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
   end

data/lib/new_relic/version.rb

@@ -12,7 +12,7 @@ module NewRelic
 
     MAJOR = 3
     MINOR = 14
-    TINY  = 2
+    TINY  = 3
 
     begin
       require File.join(File.dirname(__FILE__), 'build')

data/lib/tasks/config.html.erb

@@ -6,7 +6,7 @@
 <dl class="clamshell-list">
   <% configs.each do |config|%>
   <!-- ********** <%= config[:key] %> ********** -->
-  <dt id="<%=config[:key]%>"><%=config[:key]%></dt>
+  <dt id="<%= config[:key].gsub('.', '-') %>"><%=config[:key]%></dt>
   <dd>
   <table class="specs">
     <tbody>
@@ -18,6 +18,10 @@
         <th>Default</th>
         <td><%=config[:default]%></td>
       </tr>
+      <tr>
+        <th>Environ variable</th>
+        <td><code><%=config[:env_var]%></code></td>
+      </tr>
     </tbody>
   </table>