mmtrix_rpm 1.0.0.1

data/test/mmtrix/agent/datastores/metric_helper_test.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require File.expand_path(File.join(File.dirname(__FILE__),'..','..','..','test_helper'))
+require 'mmtrix/agent/datastores/metric_helper'
+
+module Mmtrix
+  module Agent
+    class Datastores::MetricHelperTest < Minitest::Test
+      def setup
+        @product = "JonanDB"
+        @collection = "wiggles"
+        @operation = "select"
+      end
+
+      def test_statement_metric_for
+        expected = "Datastore/statement/JonanDB/wiggles/select"
+        result = Datastores::MetricHelper.statement_metric_for(@product, @collection, @operation)
+        assert_equal expected, result
+      end
+
+      def test_operation_metric_for
+        expected = "Datastore/operation/JonanDB/select"
+        result = Datastores::MetricHelper.operation_metric_for(@product, @operation)
+        assert_equal expected, result
+      end
+
+      def test_metrics_for_in_web_context
+        Transaction.stubs(:recording_web_transaction?).returns(true)
+        expected = [
+          "Datastore/statement/JonanDB/wiggles/select",
+          "Datastore/operation/JonanDB/select",
+          "Datastore/JonanDB/allWeb",
+          "Datastore/JonanDB/all",
+          "Datastore/allWeb",
+          "Datastore/all"
+        ]
+
+        result = Datastores::MetricHelper.metrics_for(@product, @operation, @collection)
+        assert_equal expected, result
+      end
+
+      def test_metrics_for_outside_web_context
+        Transaction.stubs(:recording_web_transaction?).returns(false)
+        expected = [
+          "Datastore/statement/JonanDB/wiggles/select",
+          "Datastore/operation/JonanDB/select",
+          "Datastore/JonanDB/allOther",
+          "Datastore/JonanDB/all",
+          "Datastore/allOther",
+          "Datastore/all"
+        ]
+
+        result = Datastores::MetricHelper.metrics_for(@product, @operation, @collection)
+        assert_equal expected, result
+      end
+
+      def test_metrics_for_obeys_collection_and_operation_overrides
+        in_transaction do
+          Mmtrix::Agent.with_database_metric_name("Model", "new_method") do
+            expected = [
+              "Datastore/statement/JonanDB/Model/new_method",
+              "Datastore/operation/JonanDB/new_method",
+              "Datastore/JonanDB/allOther",
+              "Datastore/JonanDB/all",
+              "Datastore/allOther",
+              "Datastore/all"
+            ]
+
+            result = Datastores::MetricHelper.metrics_for(@product, "original_method")
+            assert_equal expected, result
+          end
+        end
+      end
+
+      def test_metrics_for_obeys_collection_override
+        in_transaction do
+          Mmtrix::Agent.with_database_metric_name("Model", nil) do
+            expected = [
+              "Datastore/statement/JonanDB/Model/original_method",
+              "Datastore/operation/JonanDB/original_method",
+              "Datastore/JonanDB/allOther",
+              "Datastore/JonanDB/all",
+              "Datastore/allOther",
+              "Datastore/all"
+            ]
+
+            result = Datastores::MetricHelper.metrics_for(@product, "original_method")
+            assert_equal expected, result
+          end
+        end
+      end
+
+      def test_metrics_ignore_overrides_for_other_products
+        in_transaction do
+          Mmtrix::Agent.with_database_metric_name("Model", "new_method", "FauxDB") do
+            expected = [
+              "Datastore/operation/JonanDB/original_method",
+              "Datastore/JonanDB/allOther",
+              "Datastore/JonanDB/all",
+              "Datastore/allOther",
+              "Datastore/all"
+            ]
+
+            result = Datastores::MetricHelper.metrics_for(@product, "original_method")
+            assert_equal expected, result
+          end
+        end
+      end
+
+      def test_metrics_applies_overrides_by_generic_product_name
+        in_transaction do
+          Mmtrix::Agent.with_database_metric_name("Model", "new_method") do
+            expected = [
+              "Datastore/statement/MoreSpecificDB/Model/new_method",
+              "Datastore/operation/MoreSpecificDB/new_method",
+              "Datastore/MoreSpecificDB/allOther",
+              "Datastore/MoreSpecificDB/all",
+              "Datastore/allOther",
+              "Datastore/all"
+            ]
+
+            result = Datastores::MetricHelper.metrics_for("MoreSpecificDB", "original_method", nil, @product)
+            assert_equal expected, result
+          end
+        end
+      end
+
+    end
+  end
+end

data/test/mmtrix/agent/datastores/mongo/metric_translator_test.rb

@@ -0,0 +1,320 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/datastores/mongo/metric_translator'
+require File.expand_path(File.join(File.dirname(__FILE__),'..','..','..','..','test_helper'))
+
+class Mmtrix::Agent::Datastores::Mongo::MetricTranslatorTest < Minitest::Test
+  include ::Mmtrix::TestHelpers::MongoMetricBuilder
+
+  def setup
+    @database_name = 'multiverse'
+    @collection_name = 'tribbles'
+  end
+
+  def test_metrics_for_insert
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:insert, { :collection => @collection_name })
+    expected = build_test_metrics(:insert)
+
+    assert_equal expected, metrics
+  end
+
+  def test_build_metrics_includes_web
+    in_web_transaction do
+      metrics = build_test_metrics('test')
+      assert_includes metrics, 'Datastore/allWeb'
+    end
+  end
+
+  def test_build_metrics_includes_other
+    in_background_transaction do
+      metrics = build_test_metrics('test')
+      assert_includes metrics, 'Datastore/allOther'
+    end
+  end
+
+  def test_build_metrics_includes_all_for_web
+    in_web_transaction do
+      metrics = build_test_metrics('test')
+      assert_includes metrics, 'Datastore/all'
+    end
+  end
+
+  def test_build_metrics_includes_all_for_other
+    in_background_transaction do
+      metrics = build_test_metrics('test')
+      assert_includes metrics, 'Datastore/all'
+    end
+  end
+
+  def test_build_metrics_is_graceful_if_exceptions_are_raised
+    Mmtrix::Agent::Datastores::MetricHelper.stubs(:metrics_for).raises("Boom")
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, {})
+    assert_empty metrics
+  end
+
+  def test_metrics_for_find
+    payload = { :database   => @database_name,
+                :collection => @collection_name,
+                :selector   => { "name" => "soterios johnson" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:find)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_find_one
+    payload = { :database   => @database_name,
+                :collection => @collection_name,
+                :selector   => {},
+                :limit      => -1 }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:findOne)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_remove
+    payload = { :database   => @database_name,
+                :collection => @collection_name,
+                :selector   => { "name" => "soterios johnson" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:remove, payload)
+    expected = build_test_metrics(:remove)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_update
+    payload = { :database   => @database_name,
+                :collection => @collection_name,
+                :selector   => { "name" => "soterios johnson" },
+                :document   => { "name" => "codemonkey" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:update, payload)
+    expected = build_test_metrics(:update)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_distinct
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { :distinct => @collection_name,
+                                 :key      => "name",
+                                 :query    => nil } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:distinct, payload)
+    expected = build_test_metrics(:distinct)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_count
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { "count"  => @collection_name,
+                                 "query"  => {},
+                                 "fields" => nil } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:count)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_group
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { "group" => { "ns"      => @collection_name,
+                                              "$reduce" => stub("BSON::Code"),
+                                              "cond"    => {},
+                                              "initial" => {:count=>0},
+                                              "key"     => {"name"=>1}}}}
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:group)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_aggregate
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   =>  { "aggregate" => @collection_name,
+                                  "pipeline" => [{"$group" => {:_id => "$says", :total => {"$sum" => 1}}}]}}
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:aggregate)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_mapreduce
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   =>  { "mapreduce" => @collection_name,
+                                  "map" => stub("BSON::Code"),
+                                  "reduce" => stub("BSON::Code"),
+                                  :out => "results"}}
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:mapreduce)
+
+    assert_equal expected, metrics
+  end
+
+
+  def test_metrics_for_find_and_modify
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { :findandmodify => @collection_name,
+                                 :query         => { "name" => "soterios johnson" },
+                                 :update        => {"name" => "codemonkey" } } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:findAndModify)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_find_and_remove
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { :findandmodify => @collection_name,
+                                 :query         => { "name" => "soterios johnson" },
+                                 :remove        => true } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:findAndRemove)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_create_index
+    payload = { :database   => @database_name,
+                :collection => "system.indexes",
+                :documents  => [ { :name => "name_1",
+                                    :ns   => "#{@database_name}.#{@collection_name}",
+                                    :key  => { "name" => 1 } } ] }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:insert, payload)
+    expected = build_test_metrics(:createIndex)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_drop_indexes
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector => { :deleteIndexes => @collection_name,
+                               :index         => "*" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:dropIndexes)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_drop_index
+    payload = { :database => @database_name,
+                :collection => "$cmd",
+                :limit => -1,
+                :selector => { :deleteIndexes => @collection_name,
+                               :index => "name_1" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:dropIndex)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_reindex
+    payload = { :database => @database_name,
+                :collection => "$cmd",
+                :limit => -1,
+                :selector => { :reIndex=> @collection_name } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:reIndex)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_drop_collection
+    payload = { :database   => @database_name,
+                :collection =>"$cmd",
+                :limit      => -1,
+                :selector   => { :drop => @collection_name } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:drop)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_rename_collection
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { :renameCollection => "#{@database_name}.#{@collection_name}",
+                                 :to=>"#{@database_name}.renamed_#{@collection_name}" } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:renameCollection)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_ismaster
+    payload = { :database   => @database_name,
+                :collection => "$cmd",
+                :limit      => -1,
+                :selector   => { :ismaster => 1 } }
+
+    @collection_name = "$cmd"
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:ismaster)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_collstats
+    payload = { :database   => @database_name,
+                :collection =>"$cmd",
+                :limit      => -1,
+                :selector   => { :collstats => @collection_name } }
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:collstats)
+
+    assert_equal expected, metrics
+  end
+
+  def test_metrics_for_unknown_command
+    payload = { :database => @database_name,
+                :collection => "$cmd",
+                :limit => -1,
+                :selector => { :mongomongomongo => @collection_name } }
+
+    @collection_name = "$cmd"
+
+    metrics = Mmtrix::Agent::Datastores::Mongo::MetricTranslator.metrics_for(:find, payload)
+    expected = build_test_metrics(:mongomongomongo)
+
+    assert_equal expected, metrics
+    assert_metrics_recorded(["Supportability/Mongo/UnknownCollection"])
+  end
+end

data/test/mmtrix/agent/datastores/mongo/obfuscator_test.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/datastores/mongo/obfuscator'
+require File.expand_path(File.join(File.dirname(__FILE__),'..','..','..','..','test_helper'))
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        class ObfuscatorTest < Minitest::Test
+          def test_obfuscator_removes_values_from_statement
+            selector = {
+              'name'     => 'soterios johnson',
+              :operation => :find,
+              :_id       => "BSON::ObjectId('?')"
+            }
+
+            expected = {
+              'name'     => '?',
+              :operation => :find,
+              :_id       => '?'
+            }
+
+            obfuscated = Obfuscator.obfuscate_statement(selector)
+            assert_equal expected, obfuscated
+          end
+
+          def test_obfuscate_selector_values_skips_whitelisted_keys
+            selector   = {
+              :benign    => 'bland data',
+              :operation => :find,
+              :_id       => "BSON::ObjectId('?')"
+            }
+
+            expected   = {
+              :benign    => 'bland data',
+              :operation => :find,
+              :_id       => '?'
+            }
+
+            obfuscated = Obfuscator.obfuscate_statement(selector, [:benign, :operation])
+            assert_equal expected, obfuscated
+          end
+
+          def test_obfuscate_nested_hashes
+            selector = {
+              "group" => {
+                "ns"      => "tribbles",
+                "$reduce" => stub("BSON::Code"),
+                "cond"    => {},
+                "initial" => { :count => 0 },
+                "key"     => { "name" => 1 }
+              }
+            }
+
+            expected = {
+              "group" => {
+                "ns"      => "?",
+                "$reduce" => "?",
+                "cond"    => {},
+                "initial" => { :count => "?" },
+                "key"     => { "name" => "?" }
+              }
+            }
+
+            obfuscated = Obfuscator.obfuscate_statement(selector)
+            assert_equal expected, obfuscated
+          end
+
+          def test_obfuscate_nested_arrays
+            selector = {
+              "aggregate" => "mongeese",
+              "pipeline"  => [{"$group"=>{:_id=>"$says", :total=>{"$sum"=>1}}}]
+            }
+
+            expected = {
+              "aggregate" => "?",
+              "pipeline"  => [{"$group"=>{:_id=>"?", :total=>{"$sum"=>"?"}}}]
+            }
+
+            obfuscated = Obfuscator.obfuscate_statement(selector)
+            assert_equal expected, obfuscated
+          end
+
+        end
+      end
+    end
+  end
+end

data/test/mmtrix/agent/datastores/mongo/statement_formatter_test.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require File.expand_path(File.join(File.dirname(__FILE__),'..','..','..','..','test_helper'))
+require 'mmtrix/agent/datastores/mongo/statement_formatter'
+
+module Mmtrix
+  module Agent
+    module Datastores
+      module Mongo
+        class StatementFormatterTest < Minitest::Test
+          DOC_STATEMENT = { :database   => "multiverse",
+                            :collection => "tribbles",
+                            :operation  => :insert,
+                            :fields     => ["field_name"],
+                            :skip       => 1,
+                            :limit      => -1,
+                            :order      => :ascending,
+
+                            :ignored    => "we're whitelisted!",
+                            :documents  => [ { "name" => "soterios johnson",
+                                               :_id   => "BSON::ObjectId()" } ] }.freeze
+
+
+          SELECTOR_STATEMENT = { :database   => 'multiverse',
+                                 :collection => 'tribbles',
+                                 :selector   => { 'name'     => 'soterios johnson',
+                                                  :operation => :find,
+                                                  :_id       => "BSON::ObjectId('?')" } }.freeze
+
+          def test_doesnt_modify_incoming_statement
+            formatted = StatementFormatter.format(DOC_STATEMENT, :find)
+            refute_same DOC_STATEMENT, formatted
+          end
+
+          def test_statement_formatter_removes_unwhitelisted_keys
+            formatted = StatementFormatter.format(DOC_STATEMENT, :find)
+            assert_equal_unordered(formatted.keys, StatementFormatter::PLAINTEXT_KEYS)
+          end
+
+          def test_can_disable_statement_capturing_queries
+            with_config(:'mongo.capture_queries' => false) do
+              formatted = StatementFormatter.format(DOC_STATEMENT, :find)
+              assert_nil formatted
+            end
+          end
+
+          def test_statement_formatter_obfuscates_by_default
+            expected = { :database   => 'multiverse',
+                         :collection => 'tribbles',
+                         :operation  => :find,
+                         :selector   => { 'name'     => '?',
+                                          :operation => :find,
+                                          :_id       => '?' } }
+
+            result = StatementFormatter.format(SELECTOR_STATEMENT, :find)
+            assert_equal expected, result
+          end
+
+          def test_statement_formatter_raw_selectors
+            with_config(:'mongo.obfuscate_queries' => false) do
+              result = StatementFormatter.format(SELECTOR_STATEMENT, :find)
+              assert_equal SELECTOR_STATEMENT.merge(:operation => :find), result
+            end
+          end
+
+        end
+      end
+    end
+  end
+end