mongo_request_logger 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+script: "bundle exec rspec"
+env:
+  - CI=true
+rvm:
+  - 1.9.3
+services:
+  - mongodb
+notifications:
+  recipients:
+    - ralf@embarkmobile.com

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+gem 'mongo'
+gem 'bson_ext'
+gem 'moped'
+gem 'rspec'
+gem 'rails', '~> 3.2.10'
+
+# Specify your gem's dependencies in mongo_request_logger.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,28 @@
+Copyright (c) 2013 Embark Mobile
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+Some code was used from the following projects:
+
+    https://github.com/le0pard/mongodb_logger (MIT License)
+

data/README.md

@@ -0,0 +1,90 @@
+# MongoRequestLogger
+
+Log requests in a structured format to MongoDB.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mongo_request_logger'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mongo_request_logger
+
+## Usage with Rails
+
+Create config/logger.yml containing something like the following:
+
+    development:
+      host: localhost
+      database: your_app_logs_production_dev
+      capsize: 100
+
+    production:
+      host: localhost
+      database: your_app_logs_production
+      username: your_app_logs_production
+      password: password
+      capsize: 1024
+
+Routes for log viewer
+
+    mount MongoRequestLogger::Viewer, :at => "log"
+
+
+
+## Usage with other Rack-based frameworks (Sinatra, etc)
+
+In config.ru, or where appropriate:
+
+    require 'moped'
+
+    config = {
+        database: 'my_log_db',
+        collection: 'log',
+        capsize: 100 * 1024 * 1024,
+    }
+
+    adapter = MongoRequestLogger::Adapters::Moped.new(config)
+    logger = MongoRequestLogger::Logger.new(adapter, "myapp.log")
+    MongoRequestLogger::Rack.logger = loggger
+
+    use MongoRequestLogger::Rack
+
+    run MyApp
+
+## Ignoring paths
+
+Often you want some requests not to be logged, for example assets files. This can be done by specifying the prefixes
+to be ignored:
+
+    MongoRequestLogger::Rack.ignore_prefixes << '/assets/'
+
+## Usage with resque
+
+Extend from MongoRequestLogger::LoggedJob, to have each job logged as if it was a request.
+
+    class MyJob
+        extend MongoRequestLogger::LoggedJob
+
+        def perform(args)
+           # ...
+        end
+    end
+
+## License
+
+All code is under the MIT licence, see LICENSE.txt.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/config.ru

@@ -0,0 +1,11 @@
+# Sinatra app for viewing logs
+
+require 'moped'
+require 'mongo_request_logger/viewer'
+
+config = {database: 'request_logger_test', collection: 'server_log', capsize: 10*1024*1024}
+
+MongoRequestLogger::Viewer.adapter = MongoRequestLogger::Adapters::Moped.new(config)
+
+MongoRequestLogger::Viewer.run!
+

data/lib/mongo_request_logger/adapters/base.rb

@@ -0,0 +1,51 @@
+# Adapted from https://github.com/le0pard/mongodb_logger
+
+module MongoRequestLogger
+  module Adapters
+    class Base
+
+      attr_reader :configuration, :connection, :connection_type, :collection, :authenticated
+
+      def collection_name
+        @configuration['collection']
+      end
+
+      def authenticated?
+        @authenticated
+      end
+
+      def check_for_collection
+        # setup the capped collection if it doesn't already exist
+        create_collection unless @connection.collection_names.include?(@configuration['collection'])
+        @collection = @connection[@configuration['collection']]
+      end
+
+      def reset_collection
+        if @connection && @collection
+          @collection.drop
+          create_collection
+        end
+      end
+
+      def reconnect
+
+      end
+
+      def collection_stats_hash(stats)
+        {
+            :is_capped => (stats["capped"] && ([1, true].include?(stats["capped"]))),
+            :count => stats["count"].to_i,
+            :size => stats["size"].to_f,
+            :storageSize => stats["storageSize"].to_f,
+            :db_name => @configuration["database"],
+            :collection => collection_name
+        }
+      end
+
+      def create_collection
+        raise "Not implemented"
+      end
+
+    end
+  end
+end

data/lib/mongo_request_logger/adapters/mongo.rb

@@ -0,0 +1,89 @@
+# Adapted from https://github.com/le0pard/mongodb_logger/
+
+require 'mongo_request_logger/adapters/base'
+
+module MongoRequestLogger
+  module Adapters
+    class Mongo < Base
+
+      def initialize(options = {})
+        @authenticated = false
+        @configuration = options.with_indifferent_access
+
+        connect
+
+        check_for_collection
+      end
+
+      def connect
+        if @configuration['url']
+          uri = URI.parse(@configuration['url'])
+          @configuration['database'] = uri.path.gsub(/^\//, '')
+          @connection ||= mongo_connection_object.db(@configuration['database'])
+          @authenticated = true
+        else
+          @connection ||= mongo_connection_object.db(@configuration['database'])
+          if @configuration['username'] && @configuration['password']
+            # the driver stores credentials in case reconnection is required
+            @authenticated = @connection.authenticate(@configuration['username'],
+                @configuration['password'])
+          end
+        end
+      end
+
+      def reconnect
+        connect
+      end
+
+      def create_collection
+        @connection.create_collection(collection_name,
+            {:capped => true, :size => @configuration['capsize'].to_i})
+      end
+
+      def create_index field
+        @collection.create_index(field)
+      end
+
+      def insert_log_record(record)
+        # TODO: we should make this non-safe?
+        @collection.insert(record)
+      end
+
+      def collection_stats
+        collection_stats_hash(@collection.stats)
+      end
+
+      def query(criteria, options={})
+        q = @collection.find(filter.get_mongo_conditions).sort('timestamp', -1)
+        if options[:limit]
+          q = q.limit(options.limit)
+        end
+        q
+      end
+
+      def find_by_id(id)
+        @collection.find_one(::BSON::ObjectId(id))
+      end
+
+      private
+
+      def mongo_connection_object
+        if @configuration['hosts']
+          conn = ::Mongo::ReplSetConnection.new(*(@configuration['hosts'] <<
+              {:connect => true, :pool_timeout => 6}))
+          @configuration['replica_set'] = true
+        elsif @configuration['url']
+          conn = ::Mongo::Connection.from_uri(@configuration['url'])
+        else
+          conn = ::Mongo::Connection.new(@configuration['host'],
+              @configuration['port'],
+              :connect => true,
+              :pool_timeout => 6)
+        end
+        @connection_type = conn.class
+        conn
+      end
+
+    end
+  end
+end

data/lib/mongo_request_logger/adapters/moped.rb

@@ -0,0 +1,94 @@
+# Adapted from https://github.com/le0pard/mongodb_logger/
+
+require 'mongo_request_logger/adapters/base'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module MongoRequestLogger
+  module Adapters
+    class Moped < Base
+
+      def initialize(options = {})
+        @configuration = options.with_indifferent_access
+        @configuration['host'] ||= '127.0.0.1'
+        @configuration['port'] ||= '27017'
+        @configuration['collection'] ||= 'server_log'
+
+        if @configuration['url']
+          uri = URI.parse(@configuration['url'])
+          @configuration['database'] = uri.path.gsub(/^\//, '')
+          @connection ||= mongo_connection_object
+          @connection.use @configuration['database']
+          @authenticated = true
+        else
+          @connection ||= mongo_connection_object
+          @connection.use @configuration['database']
+
+          login
+        end
+
+        check_for_collection
+      end
+
+      def login
+        if @configuration['username'] && @configuration['password']
+          # the driver stores credentials in case reconnection is required
+          @authenticated = @connection.login(@configuration['username'],
+              @configuration['password'])
+        end
+
+        # Allow MongoDB configured both with and without authentication
+        begin
+          @connection.command(ping: 1)
+        rescue ::Moped::Errors::AuthenticationFailure => e
+          @connection.logout
+          @connection.command(ping: 1)
+        end
+      end
+
+      def create_collection
+        @connection.command(create: collection_name, capped: true, size:  @configuration['capsize'].to_i)
+      end
+
+      def create_index field
+        @collection.indexes.create({field => 1})
+      end
+
+      def insert_log_record(record)
+        record[:_id] = ::Moped::BSON::ObjectId.new
+        @connection.with(safe: false)[collection_name].insert(record)
+      end
+
+      def collection_stats
+        collection_stats_hash(@connection.command(collStats: collection_name))
+      end
+
+      def query(criteria, options={})
+        q = @collection.find(criteria).sort({'timestamp' => -1})
+        if options[:limit]
+          q = q.limit(options[:limit])
+        end
+        q
+      end
+
+      def find_by_id(id)
+        @collection.find("_id" => ::Moped::BSON::ObjectId.from_string(id)).first
+      end
+
+      private
+
+      def mongo_connection_object
+        if @configuration['hosts']
+          conn = ::Moped::Session.new(@configuration['hosts'].map{|(host,port)| "#{host}:#{port}"}, :timeout => 6)
+          @configuration['replica_set'] = true
+        elsif @configuration['url']
+          conn = ::Moped::Session.connect(@configuration['url'])
+        else
+          conn = ::Moped::Session.new(["#{@configuration['host']}:#{@configuration['port']}"], :timeout => 6)
+        end
+        @connection_type = conn.class
+        conn
+      end
+
+    end
+  end
+end

data/lib/mongo_request_logger/backtrace_cleaner.rb

@@ -0,0 +1,11 @@
+module MongoRequestLogger
+  module BacktraceCleaner
+    def self.clean backtrace
+      if defined? Rails and Rails.respond_to? :backtrace_cleaner
+        Rails.backtrace_cleaner.clean(backtrace)
+      else
+        backtrace
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/config.rb

@@ -0,0 +1,109 @@
+require 'yaml'
+
+module MongoRequestLogger
+  class Config
+    attr_accessor :raw_config
+    attr_accessor :params
+    attr_accessor :namespace
+
+    def initialize(config=nil)
+      @raw_config = {}
+      @params = {}
+      @namespace = nil
+      load_config(config) unless config.nil?
+    end
+
+    def namespaced(namespace)
+      config = deep_merge(@raw_config['common'], @raw_config[namespace])
+      ac = self.class.new(config)
+      ac.namespace = namespace
+      ac
+    end
+
+
+    def namespaces
+      @raw_config.keys - %w(common)
+    end
+
+    def load_file(file)
+      begin
+        hash = YAML::load(IO.read(file))
+      rescue => e
+        hash = nil
+      end
+      load_config(hash) unless hash.nil?
+    end
+
+    def deep_merge(a, b)
+      if a.is_a? Hash and b.is_a? Hash
+        r = a.dup
+        for k, v in b
+          r[k] = deep_merge(a[k], v)
+        end
+        r
+      elsif b.is_a? Hash
+        b.dup
+      else
+        b
+      end
+    end
+
+    def load_config(config)
+      @raw_config = deep_merge(@raw_config, config)
+      @params = {}
+      @raw_config.each { |k, v|
+        if v.is_a? Hash
+          @params[k] = self.class.new(v)
+        else
+          @params[k] = v
+        end
+      }
+
+      create_accessors!
+    end
+
+    def has?(param)
+      @params.key?(param.to_s)
+    end
+
+    def get(param)
+      @params[param.to_s]
+    end
+
+    def [](param)
+      get(param)
+    end
+
+    def []=(param, value)
+      @params[param.to_s] = value
+    end
+
+    def with_indifferent_access
+      self
+    end
+
+    def create_accessors!
+      @params.each do |key,val|
+        create_accessor_for(key)
+      end
+    end
+
+    # Use instance_eval/class_eval because they're actually more efficient than define_method{}
+    # http://stackoverflow.com/questions/185947/ruby-definemethod-vs-def
+    # http://bmorearty.wordpress.com/2009/01/09/fun-with-rubys-instance_eval-and-class_eval/
+    def create_accessor_for(key)
+      key = key.to_s
+      return unless key =~ /^[\w_]+$/  # could have "some-setting:" which blows up eval
+
+      self.class.class_eval <<-EndEval
+        def #{key}
+          return @params["#{key}"]
+        end
+
+        def #{key}= value
+          @params["#{key}"] = value
+        end
+      EndEval
+    end
+  end
+end

data/lib/mongo_request_logger/ext/buffered_logger.rb

@@ -0,0 +1,6 @@
+require 'active_support/buffered_logger'
+require 'mongo_request_logger/logger_extensions'
+
+class ActiveSupport::BufferedLogger
+  include MongoRequestLogger::LoggerExtensions
+end

data/lib/mongo_request_logger/ext/logger.rb

@@ -0,0 +1,6 @@
+require 'logger'
+require 'mongo_request_logger/logger_extensions'
+
+class Logger
+  include MongoRequestLogger::LoggerExtensions
+end