mongo_profiler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a47fc67c8b76da256d794a635b3e719f783664aa
+  data.tar.gz: 19b951a31adeebadf0e38de08b2768bac0111e58
+SHA512:
+  metadata.gz: 8835e0823b0f5b7b008dc714178a3e4412bcc36f6d00eb2eb4b49f1b71fa70f143bc2f261e88bb8b8a9e66cfc5db0514d7c752f6d5407daa5064d1f5080a99f3
+  data.tar.gz: 4889bf3ffe0465f163b0a13d977747956d2df0c61f3ece6ff38cabdcedef770195768350f7140147209e7c4eed2c262b9a672f3ec27a3e112c8a1e7fbc51a260

data/.gitignore

@@ -0,0 +1 @@
+web/assets/bower_components

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format d

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mongo-ruby-profiler.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,78 @@
+PATH
+  remote: .
+  specs:
+    mongo_profiler (0.0.1)
+      activesupport
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.0.2)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    atomic (1.1.14)
+    bson (1.9.2)
+    bson_ext (1.9.2)
+      bson (~> 1.9.2)
+    byebug (2.5.0)
+      columnize (~> 0.3.6)
+      debugger-linecache (~> 1.2.0)
+    coderay (1.1.0)
+    columnize (0.3.6)
+    debugger-linecache (1.2.0)
+    diff-lcs (1.2.5)
+    i18n (0.6.9)
+    method_source (0.8.2)
+    minitest (4.7.5)
+    mongo (1.9.2)
+      bson (~> 1.9.2)
+    multi_json (1.8.4)
+    pry (0.9.12.4)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry-byebug (1.2.1)
+      byebug (~> 2.2)
+      pry (~> 0.9.12)
+    rack (1.5.2)
+    rack-protection (1.5.1)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rake (10.1.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    shotgun (0.9)
+      rack (>= 1.0)
+    sinatra (1.4.4)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.7)
+    thread_safe (0.1.3)
+      atomic
+    tilt (1.4.1)
+    tzinfo (0.3.38)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bson_ext
+  mongo (= 1.9.2)
+  mongo_profiler!
+  pry-byebug
+  rack-test
+  rake
+  rspec (~> 2.14.1)
+  shotgun
+  sinatra

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Pablo Cantero
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# Mongo Profiler
+
+Database profiling tools are awesome and always useful. I love [Mongo Profiling](http://docs.mongodb.org/manual/tutorial/manage-the-database-profiler/). But unfortunately these tools don't match queries and code they are profiling, so sometimes isn't easy to match where the slow queries are performed.
+
+The Mongo Profiler is a <del>refinement</del> patch in the [mongo-ruby-driver](https://github.com/mongodb/mongo-ruby-driver) to log all execute queries and their respective callers in a [capped collections](http://docs.mongodb.org/manual/core/capped-collections/).
+
+It isn't competitor for the Mongo's built-in profiling, it is just a complementary tool to help us to profile our queries.
+
+An interesting feature in the Mongo Profiler is that we can group queries by "life cycles". For example, in a web application it can be the `request_id`, so you will be able to see how many queries, how long did they take, the explain plans etc for a specific request.
+
+First time I used it, I was shocked to see some pages doing lot of duplicated queries, even though some were really fast, they were unnecessary, I could get rid of some of them just by "memorising" some documents.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mongo_profiler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mongo_profiler
+
+## Usage
+
+### Rails application
+
+```ruby
+# config/initializers/mongo_profiler_setup.rb
+
+require 'mongo_profiler'
+require 'mongo_profiler/extensions/mongo/cursor'
+
+MongoProfiler.connect('localhost', 27017, 'my_database')
+
+MongoProfiler.application_name = 'my_application'
+
+# To enable Statsd
+# MongoProfiler.stats_client = MyStatsdClientInstance
+
+# To show graphite graphs
+# MongoProfiler.graphite_url = 'http://my_graphite'
+```
+
+```ruby
+# app/controllers/application_controller.rb
+
+class ApplicationController < ActionController::Base
+  before_filter :mongo_profiler_setup
+
+  private
+
+  def mongo_profiler_setup
+    # aggregate queries by request
+    MongoProfiler.group_id = "request-#{request.uuid}"
+
+    # to show the request url
+    MongoProfiler.extra_attrs[:request_url]  = request.url
+  rescue => e
+    p "MongoProfiler: #{e.message}"
+  end
+end
+```
+
+```ruby
+# config/routes.rb
+
+require 'mongo_profiler/web'
+
+MyApplication::Application.routes.draw do
+  mount MongoProfiler::Web => '/mongo_profiler'
+
+  # Security with Devise
+  # authenticate :user do
+  #  mount MongoProfiler::Web => '/mongo_profiler'
+  # end
+  #
+  # authenticate :user, lambda { |u| u.admin? } do
+  #  mount MongoProfiler::Web => '/mongo_profiler'
+  # end
+end
+```
+
+## Screenshots
+
+### Dashboard index
+
+![Dashboard Index](https://raw.github.com/phstc/mongo_profiler/master/assets/mongo_profiler_dashboard_index.png)
+
+### Queries Group Index
+
+![Queries Group Index](https://raw.github.com/phstc/mongo_profiler/master/assets/mongo_profiler_group_details.png)
+
+### Query details
+
+![Query Details](https://raw.github.com/phstc/mongo_profiler/master/assets/mongo_profiler_query_details.png)
+
+### Query details (backtrace)
+
+![Query Details Backtrace](https://raw.github.com/phstc/mongo_profiler/master/assets/mongo_profiler_query_details_backtrace.png)
+
+
+## Contributing
+
+1. Fork it ( http://github.com/phstc/mongo_profiler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/config.ru

@@ -0,0 +1,13 @@
+require 'pry-byebug'
+require 'mongo'
+require 'mongo_profiler'
+
+require 'mongo_profiler/web'
+
+CONNECTION = Mongo::MongoClient.new
+DB         = CONNECTION.db('mongo_profiler-database')
+COLL       = DB['example-collection']
+
+MongoProfiler.connect('localhost', 27017, 'mongo_profiler-database')
+
+run MongoProfiler::Web

data/lib/mongo_profiler.rb

@@ -0,0 +1,108 @@
+require 'mongo'
+require 'json'
+require 'active_support/core_ext/hash/indifferent_access'
+
+require 'mongo_profiler/version'
+require 'mongo_profiler/profiler'
+require 'mongo_profiler/caller'
+require 'mongo_profiler/payload'
+require 'mongo_profiler/stats'
+
+module MongoProfiler
+  COLLECTION_CONFIG_NAME   = 'mongo_profiler_config'
+  COLLECTION_PROFILER_NAME = 'mongo_profiler'
+
+  class << self
+    attr_accessor :extra_attrs,
+      :group_id,
+      :application_name,
+      :stats_client,
+      :stats_prefix,
+      :graphite_url
+
+    attr_reader :database
+
+    def log(document)
+      collection.insert(document.merge(application_name:  MongoProfiler.application_name,
+                                       group_id:          MongoProfiler.group_id))
+    end
+
+    def should_skip?(payload, _caller)
+      Payload.new(payload).system_any? || _caller.mongo_profiler_caller?
+    end
+
+    def disable!
+      # maybe we can refactor to check if the collections exist before trying to create them.
+      # we must make sure the collections are in place before enabled/disable otherwise mongo will create a normal collection,
+      # not a capped one, breaking the disable & enable functionality
+      create_collections
+
+      collection_config.insert(enabled: false)
+    end
+
+    def enable!
+      # check `disable!` comment
+      create_collections
+
+      collection_config.insert(enabled: true)
+    end
+
+    def enabled?
+      !!collection_config.find.first.to_h['enabled']
+    end
+
+    def disabled?
+      !enabled?
+    end
+
+    def extra_attrs
+      @extra_attrs ||= {}
+    end
+
+    def group_id
+      # The group_id is used to determine the life cycle where the queries occurred.
+      # For web applications a life cycle can be a request.
+      # So people can filter all Mongo Queries per request based on request#url and/or request#uuid.
+      @group_id ||= { process_pid:       Process.pid,
+                      thread_object_id:  Thread.current.object_id }.to_a.join('-')
+    end
+
+    def stats_client=(stats_client)
+      @stats_client = MongoProfiler::Stats.new(stats_client)
+    end
+
+    def create_collections
+      # http://docs.mongodb.org/manual/core/capped-collections/
+      # 1_048_576 - 1MB - allows only one document (max: 1)
+      @database.create_collection(COLLECTION_CONFIG_NAME, capped: true, size: 1_048_576, max: 1)
+
+      # 4_001_792 - 3.82MB - same size as db.system.profile.stats()
+      @database.create_collection(COLLECTION_PROFILER_NAME, capped: true, size: 4_001_792, max: 9223372036854775807)
+    end
+
+    def collection
+      @collection ||= @database[COLLECTION_PROFILER_NAME]
+    end
+
+    def collection_config
+      @collection_config ||= @database[COLLECTION_CONFIG_NAME]
+    end
+
+    def connected?
+      !!(@connection && @database)
+    end
+
+    def connect(host = 'localhost', port = 27017, db = nil, user = nil, pass = nil, options = {})
+      @connection, @database = nil
+
+      @connection = Mongo::MongoClient.new(host, port, options)
+      if db
+        @database = @connection.db(db)
+      else
+        # default database
+        @database = @connection.db
+      end
+      @database.authenticate(user, pass) if user && pass
+    end
+  end
+end

data/lib/mongo_profiler/caller.rb

@@ -0,0 +1,27 @@
+module MongoProfiler
+  class Caller
+    attr_reader :file, :line, :method, :_caller
+
+    def initialize(_caller)
+      @_caller = _caller.dup
+
+      caller_head = project_callers[0].split ':'
+
+      # i.e. "/Users/pablo/workspace/project/spec/mongo_profiler_spec.rb:7:in `new'",
+      @file   = caller_head[0]
+      @line   = caller_head[1].to_i
+      @method = project_callers[0][/`.*'/][1..-2]
+    end
+
+    def mongo_profiler_caller?
+      _caller.any? { |line| line.include?('mongo_profiler') && !line.include?('_spec') }
+    end
+
+    private
+
+    def project_callers
+      # skip gem/bundle entries
+      @project_callers ||= _caller.select { |line| !line.include?('bundle/ruby') && !line.include?('gem/ruby') && !line.include?('rubies/ruby')  }
+    end
+  end
+end

data/lib/mongo_profiler/extensions/mongo/cursor.rb

@@ -0,0 +1,38 @@
+Mongo::Cursor.class_eval do
+  alias_method :original_send_initial_query, :send_initial_query
+
+  def send_initial_query
+    beginning_time = Time.now
+    original_send_initial_query
+    total_time = Time.now - beginning_time
+    begin
+      _caller = MongoProfiler::Caller.new(caller)
+
+      return if MongoProfiler.should_skip?(instrument_payload, _caller) || MongoProfiler.disabled?
+
+      result = {}
+
+      result[:total_time] = total_time
+
+      # the payload sent to mongo
+      result[:instrument_payload] = JSON.dump(instrument_payload)
+
+      result[:file]   = _caller.file
+      result[:line]   = _caller.line
+      result[:method] = _caller.method
+
+      result[:extra_attrs] = MongoProfiler.extra_attrs
+
+      # TODO rename `_caller` object instance to something more meaningful in this context
+      result[:backtrace]  = _caller._caller
+
+      MongoProfiler.log(result)
+
+      if stats_client = MongoProfiler.stats_client
+        stats_client.populate(_caller, total_time)
+      end
+    rescue => e
+      p "MongoProfiler: #{e.message}"
+    end
+  end
+end

data/lib/mongo_profiler/payload.rb

@@ -0,0 +1,33 @@
+module MongoProfiler
+  class Payload
+    attr_reader :payload
+
+    def initialize(payload)
+      @payload = (payload || {}).dup.with_indifferent_access
+    end
+
+    def system_database?
+      !payload['database'].to_s.match(/^admin|system/).nil?
+    end
+
+    def system_collection?
+      !payload['collection'].to_s.match(/^mongo_|system/).nil?
+    end
+
+    def system_count?
+      !payload['selector'].to_h['count'].to_s.match(/^mongo_|system/).nil?
+    end
+
+    def system_distinct?
+      !payload['selector'].to_h['distinct'].to_s.match(/^mongo_|system/).nil?
+    end
+
+    def system_command?
+      payload['collection'] == '$cmd' && !(payload['selector'].to_h.has_key?('count') || payload['selector'].to_h.has_key?('distinct'))
+    end
+
+    def system_any?
+      system_database? || system_collection? || system_count? || system_distinct? || system_command?
+    end
+  end
+end