newrelic_rpm 3.18.0.329 → 3.18.1.330

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c1e2869b811ed3b76c7a71c8386d238e53fef6b
-  data.tar.gz: cdd29e0fce09cca5e0c72d07a2471b33ffe57282
+  metadata.gz: aea433d0e5dddee05b76a21d682702ef9b6858e1
+  data.tar.gz: f9bd9e91d487865e5d33630eda81c49cadb29757
 SHA512:
-  metadata.gz: 1207f33db770af8824c989fea1658d55ac2a9e1630f2fea50044a498908257e6d51081aaecaae94f09a8238ceba070b7b3744026f390c98e377b688a2c59c3ea
-  data.tar.gz: 61ee001c5fc41aabfdb89b3ba741bfc9a37eb3b74bb8b16ef1e374f918dd3baba9ab1308e968c38c89b146a1720af347324e808031f968f365f16135cfc4b7c2
+  metadata.gz: 41eb55392a0eb008ef66bd77979e15cd45617536cbe81e331a36cf86a1f1606301250bb538b2b48049b057818116e90bc8e5bf5d1c4d191c4225b240bc99d506
+  data.tar.gz: 537960507a71710172aaa29442595c172559921fe7ee8c4c071a1f864e7f7e4917042c10442decd3a169008abf3ebce2755d1432c418eab519d1e7c3ae7ed9b8

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # New Relic Ruby Agent Release Notes #
 
+## v3.18.1 ##
+
+  * Ensure Mongo aggregate queries are properly obfuscated
+
+  Instrumentation for the Mongo 2.x driver had a bug where the `pipeline`
+  attribute of Mongo aggregate queries was not properly obfuscated. Users
+  who have sensitive data in their `aggregate` queries are strongly encouraged
+  to upgrade to this version of the agent. Users who are unable to upgrade are
+  encouraged to turn off query collection using by setting
+  `mongo.capture_queries` to false in their newrelic.yml files.
+
+  * Early access Redis 4.0 instrumentation
+
+  Our Redis instrumentation has been tested against Redis 4.0.0.rc1.
+
 ## v3.18.0 ##
 
   * Ruby 2.4.0 support

data/lib/new_relic/agent/datastores/mongo/event_formatter.rb

@@ -10,7 +10,7 @@ module NewRelic
         module EventFormatter
 
           # Keys that will get their values replaced with '?'.
-          OBFUSCATE_KEYS = [ 'filter', 'query' ].freeze
+          OBFUSCATE_KEYS = [ 'filter', 'query', 'pipeline' ].freeze
 
           # Keys that will get completely removed from the statement.
           BLACKLISTED_KEYS = [ 'deletes', 'documents', 'updates' ].freeze

data/lib/new_relic/agent/datastores/mongo/obfuscator.rb

@@ -11,25 +11,30 @@ module NewRelic
           WHITELIST = [:operation].freeze
 
           def self.obfuscate_statement(source, whitelist = WHITELIST)
-            obfuscated = {}
-            source.each do |key, value|
-              if whitelist.include?(key)
-                obfuscated[key] = value
-              else
-                obfuscated[key] = obfuscate_value(value, whitelist)
+            if source.is_a? Hash
+              obfuscated = {}
+              source.each do |key, value|
+                if whitelist.include?(key)
+                  obfuscated[key] = value
+                else
+                  obfuscated[key] = obfuscate_value(value, whitelist)
+                end
               end
+              obfuscated
+            else
+              obfuscate_value(source, whitelist)
             end
-
-            obfuscated
           end
 
+          QUESTION_MARK = '?'.freeze
+
           def self.obfuscate_value(value, whitelist = WHITELIST)
             if value.is_a?(Hash)
               obfuscate_statement(value, whitelist)
             elsif value.is_a?(Array)
               value.map {|v| obfuscate_value(v, whitelist)}
             else
-              '?'
+              QUESTION_MARK
             end
           end
         end

data/lib/new_relic/version.rb

@@ -12,7 +12,7 @@ module NewRelic
 
     MAJOR = 3
     MINOR = 18
-    TINY  = 0
+    TINY  = 1
 
     begin
       require File.join(File.dirname(__FILE__), 'build')

data/test/environments/norails/Gemfile

@@ -12,12 +12,11 @@ else
 end
 
 gem 'rack-test'
-gem 'json', '< 2.0.0' if RUBY_VERSION == "1.8.7"
+gem 'json', '< 1.8.5' if RUBY_VERSION == "1.8.7"
 
 platforms :rbx do
   gem "rubysl"
   gem "rubysl-test-unit"
-  gem "json", '< 2.0.0'
   gem "psych"
   gem "racc"  # https://github.com/rubinius/rubinius/issues/2632
 end

data/test/environments/rails21/Gemfile

@@ -24,7 +24,8 @@ gem 'hometown', '~> 0.2.5'
 
 if RUBY_VERSION < '1.9'
   gem 'git', '< 1.3' # git 1.3.0 requires Ruby version >= 1.9
-  gem 'json_pure', '< 2.0'
+  gem 'json_pure', '< 1.8.5'
+  gem 'json', '< 1.8.5'
   gem 'rdoc', '4.2.0'
 end
 

data/test/environments/rails22/Gemfile

@@ -25,7 +25,8 @@ gem 'hometown', '~> 0.2.5'
 
 if RUBY_VERSION < '1.9'
   gem 'git', '< 1.3' # git 1.3.0 requires Ruby version >= 1.9
-  gem 'json_pure', '< 2.0'
+  gem 'json_pure', '< 1.8.5'
+  gem 'json', '< 1.8.5'
   gem 'rdoc', '4.2.0'
 end
 

data/test/environments/rails23/Gemfile

@@ -7,6 +7,7 @@ gem 'minitest', '~>4.7.5'
 gem 'mocha', :require => false
 gem 'rack', '< 2.0.0'
 gem 'rack-test'
+gem 'json', '< 1.8.5' if RUBY_VERSION == "1.8.7"
 
 if RUBY_VERSION < '2.0'
   gem 'rdoc', '4.2.0'

data/test/environments/rails30/Gemfile

@@ -7,6 +7,7 @@ gem 'minitest', '~>4.7.5'
 gem 'mocha', :require => false
 gem 'rack', '< 2.0.0'
 gem 'rack-test'
+gem 'json', '< 1.8.5' if RUBY_VERSION == "1.8.7"
 
 gem 'newrelic_rpm', :path => '../../..'
 

data/test/environments/rails31/Gemfile

@@ -8,6 +8,7 @@ gem 'minitest', '~>4.7.5'
 gem 'mocha', :require => false
 gem 'rack', '< 2.0.0'
 gem 'rack-test'
+gem 'json', '< 1.8.5' if RUBY_VERSION == "1.8.7"
 
 gem 'newrelic_rpm', :path => '../../..'
 

data/test/environments/rails32/Gemfile

@@ -8,6 +8,7 @@ gem 'minitest', '~>4.7.5'
 gem 'mocha', :require => false
 gem 'rack', '< 2.0.0'
 gem 'rack-test'
+gem 'json', '< 1.8.5' if RUBY_VERSION == "1.8.7"
 
 gem "newrelic_rpm", :path => "../../.."
 

data/test/multiverse/lib/multiverse/suite.rb

@@ -177,7 +177,7 @@ module Multiverse
         f.puts minitest_line unless gemfile_text =~ /^\s*gem .minitest[^_]./
         f.puts rake_line unless gemfile_text =~ /^\s*gem .rake[^_]./ || suite == 'rake'
         if RUBY_VERSION == "1.8.7"
-          f.puts "gem 'json', '< 2.0.0'" unless gemfile_text =~ /^\s.*gem .json./
+          f.puts "gem 'json', '< 1.8.5'" unless gemfile_text =~ /^\s.*gem .json./
         end
 
         rbx_gemfile_lines(f, gemfile_text)
@@ -297,6 +297,8 @@ module Multiverse
       case
       when RUBY_VERSION >= '2.4'
         '~> 2.0.2'
+      when RUBY_VERSION == '1.8.7'
+        '< 1.8.5'
       else
         '< 2.0.0'
       end

data/test/multiverse/suites/datamapper/Envfile

@@ -11,7 +11,6 @@ adapter_gems = <<-RB
 RB
 
 gemfile <<-RB
-  gem 'json', '>= 1.8.1'
   gem 'datamapper', '~> 1.2.0', :require => 'data_mapper'
   gem 'dm-ar-finders', '~> 1.2.0'
   gem 'addressable', '2.3.8', :require => 'addressable/uri' # addressable >= 2.4.0 does not support Ruby 1.8.7

data/test/multiverse/suites/json/Envfile

@@ -1,5 +1,5 @@
 after_2_0_0  = [ '~> 2.0.2' ]
-before_2_4_0 = [ '~> 1.8.1' ]
+before_2_4_0 = [ '< 1.8.5' ]
 before_2_2_0 = [ '~> 1.7.7', '~> 1.6.8', '~> 1.5.5' ]
 before_2_0_0 = [ '~> 1.4.6', '~> 1.2.4', '~> 1.1.9' ]
 before_1_9_2 = [ '1.1.2' ]

data/test/multiverse/suites/mongo/mongo2_instrumentation_test.rb

@@ -24,7 +24,7 @@ if NewRelic::Agent::Datastores::Mongo.is_supported_version? &&
             @collection_name = "tribbles-#{SecureRandom.hex(16)}"
             @collection = @database.collection(@collection_name)
 
-            @tribbles = [{'name' => 'soterios johnson'}, {'name' => 'wes mantooth'}]
+            @tribbles = [{'name' => 'soterios johnson', 'count' => 1}, {'name' => 'wes mantooth', 'count' => 2}]
 
             NewRelic::Agent::Transaction.stubs(:recording_web_transaction?).returns(true)
             NewRelic::Agent.drop_buffered_data
@@ -205,6 +205,54 @@ if NewRelic::Agent::Datastores::Mongo.is_supported_version? &&
             assert_metrics_recorded(expected)
           end
 
+          def test_records_metrics_for_aggregate
+            in_transaction do
+              @collection.aggregate([
+                {'$group' => {'_id' => "name", "max" => {'$max'=>"$count"}}},
+                {'$match' => {'max' => {'$gte' => 1}}}
+              ]).to_a
+            end
+
+            metrics = build_test_metrics(:aggregate, true)
+            expected = metrics_with_attributes(metrics)
+
+            assert_metrics_recorded(expected)
+          end
+
+          def test_aggregate_pipeline_obfuscated_by_default
+            in_transaction do
+              @collection.aggregate([
+                {'$group' => {'_id' => "name", "max" => {'$max'=>"$count"}}},
+                {'$match' => {'max' => {'$gte' => 1}}}
+              ]).to_a
+            end
+
+            sample = NewRelic::Agent.instance.transaction_sampler.last_sample
+            metric = "Datastore/statement/MongoDB/#{@collection_name}/aggregate"
+            node = find_node_with_name(sample, metric)
+
+            expected = [
+              {"$group"=>{"_id"=>"?", "max"=>{"$max"=>"?"}}},
+              {"$match"=>{"max"=>{"$gte"=>"?"}}}
+            ]
+
+            assert_equal expected, node[:statement]["pipeline"]
+          end
+
+          def test_filter_obfuscated_by_default
+            in_transaction do
+              @collection.find("name" => "Wes Mantooth", "count" => {"$gte" => 1}).to_a
+            end
+
+            sample = NewRelic::Agent.instance.transaction_sampler.last_sample
+            metric = "Datastore/statement/MongoDB/#{@collection_name}/find"
+            node = find_node_with_name(sample, metric)
+
+            expected = {"name"=>"?", "count"=>{"$gte"=>"?"}}
+
+            assert_equal expected, node[:statement]["filter"]
+          end
+
           def test_batched_queries
             25.times do |i|
               @collection.insert_one :name => "test-#{i}", :active => true

data/test/multiverse/suites/redis/Envfile

@@ -1,3 +1,10 @@
+if RUBY_VERSION >= '2.1'
+  gemfile <<-RB
+    gem 'rack'
+    gem 'redis', '~>4.0.0.rc1'
+  RB
+end
+
 gemfile <<-RB
   gem 'rack'
   gem 'redis', '~>3.2.1'

data/test/multiverse/suites/redis/redis_instrumentation_test.rb

@@ -249,7 +249,7 @@ class NewRelic::Agent::Instrumentation::RedisInstrumentationTest < Minitest::Tes
 
   def test_records_hostname_on_tt_node_for_get_with_unix_domain_socket
     redis = Redis.new
-    redis.client.stubs(:path).returns('/tmp/redis.sock')
+    redis.send(client).stubs(:path).returns('/tmp/redis.sock')
 
     in_transaction do
       redis.get("foo")
@@ -279,7 +279,7 @@ class NewRelic::Agent::Instrumentation::RedisInstrumentationTest < Minitest::Tes
 
   def test_records_hostname_on_tt_node_for_multi_with_unix_domain_socket
     redis = Redis.new
-    redis.client.stubs(:path).returns('/tmp/redis.sock')
+    redis.send(client).stubs(:path).returns('/tmp/redis.sock')
 
     in_transaction do
       redis.multi do
@@ -296,7 +296,7 @@ class NewRelic::Agent::Instrumentation::RedisInstrumentationTest < Minitest::Tes
 
   def test_records_unknown_unknown_metric_when_error_gathering_instance_data
     redis = Redis.new
-    redis.client.stubs(:path).raises StandardError.new
+    redis.send(client).stubs(:path).raises StandardError.new
     in_transaction do
       redis.get("foo")
     end
@@ -319,5 +319,13 @@ class NewRelic::Agent::Instrumentation::RedisInstrumentationTest < Minitest::Tes
     assert_equal ['bar', 'bat'], @redis.pipelined { @redis.get('foo'); @redis.get('baz') }
     assert_equal 2, @redis.del('foo', 'baz')
   end
+
+  def client
+    if NewRelic::VersionNumber.new(Redis::VERSION).major_version < 4
+      :client
+    else
+      :_client
+    end
+  end
 end
 end

data/test/new_relic/agent/datastores/mongo/event_formatter_test.rb

@@ -56,6 +56,14 @@ module NewRelic
             "deletes" => [{ :q => { :_id => { "$gt" => 1 }}, :limit => 1 }]
           }.freeze
 
+          AGGREGATE_COMMAND = {
+            "aggregate"=>"tribbles",
+            "pipeline"=>[
+              {"$group"=> {"_id"=>"name", "max"=>{"$max"=>"$count"}}},
+              {"$match"=>{"max"=>{"$gte"=>1}}}
+            ]
+          }
+
           if RUBY_VERSION > "1.9.3"
 
             def test_doesnt_modify_incoming_statement
@@ -146,6 +154,22 @@ module NewRelic
               formatted = EventFormatter.format(:delete, DATABASE, DELETE_COMMAND)
               assert_equal expected, formatted
             end
+
+            def test_event_formatter_obfuscates_pipeline
+              expected = {
+                :operation => :aggregate,
+                :database => DATABASE,
+                :collection => "tribbles",
+                "aggregate" => "tribbles",
+                "pipeline" => [
+                  {"$group" => {"_id" => "?", "max" => {"$max" => "?"}}},
+                  {"$match" => {"max" => {"$gte" => "?"}}}
+                ]
+              }
+
+              formatted = EventFormatter.format(:aggregate, DATABASE, AGGREGATE_COMMAND)
+              assert_equal expected, formatted
+            end
           end
         end
       end

data/test/new_relic/agent/datastores/mongo/obfuscator_test.rb

@@ -69,6 +69,14 @@ module NewRelic
             assert_equal expected, obfuscated
           end
 
+          def test_obfuscates_array_statement
+            statement = [{"$group"=>{:_id=>"$says", :total=>{"$sum"=>1}}}]
+            expected = [{"$group"=>{:_id=>"?", :total=>{"$sum"=>"?"}}}]
+
+            obfuscated = Obfuscator.obfuscate_statement(statement)
+            assert_equal expected, obfuscated
+          end
+
           def test_obfuscate_nested_arrays
             selector = {
               "aggregate" => "mongeese",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: newrelic_rpm
 version: !ruby/object:Gem::Version
-  version: 3.18.0.329
+  version: 3.18.1.330
 platform: ruby
 authors:
 - Tim Krajcar
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-10 00:00:00.000000000 Z
+date: 2017-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake