mongodb_logger 0.2.6-jruby

data/features/step_definitions/rails_application_steps.rb

@@ -0,0 +1,65 @@
+require 'active_support/core_ext/string/inflections'
+
+Given /^I have built and installed the "([^\"]*)" gem$/ do |gem_name|
+  @terminal.build_and_install_gem(File.join(PROJECT_ROOT, "#{gem_name}.gemspec"))
+end
+
+When /^I generate a new Rails application$/ do
+  @terminal.cd(TEMP_DIR)
+  version_string = ENV['RAILS_VERSION']
+  rails_create_command = 'new'
+  
+  rails_dir_search = (version_string =~ /^3.(0|1)/) ? "rails" : "railties"
+  
+  load_rails = <<-RUBY
+    gem "rails", "#{version_string}"; \
+    load Gem.bin_path("#{rails_dir_search}", "rails", "#{version_string}")
+  RUBY
+
+  @terminal.run(%{ruby -rrubygems -rthread -e "#{load_rails.gsub("\"", "\\\"").strip!}" #{rails_create_command} rails_root})
+  if rails_root_exists?
+    @terminal.echo("Generated a Rails #{version_string} application")
+  else
+    raise "Unable to generate a Rails application:\n#{@terminal.output}"
+  end
+  #require_thread if rails30?
+end
+
+When /^I configure my application to require the "([^\"]*)" gem(?: with version "(.+)")?$/ do |gem_name, version|
+  bundle_gem(gem_name, version)
+end
+
+When /^I setup mongodb_logger tests$/ do
+  copy_tests
+  add_routes
+end
+
+When /^I setup all gems for rails$/ do
+  bundle_gem("therubyracer", nil) if rails31?
+  step %{I run "bundle install"}
+  @terminal.status.exitstatus.should == 0
+end
+
+When /^I prepare rails environment for testing$/ do
+  step %{I run "rake db:create db:migrate RAILS_ENV=test --trace"}
+  @terminal.status.exitstatus.should == 0
+end
+
+
+Then /^the tests should have run successfully$/ do
+  step %{I run "rake test RAILS_ENV=test --trace"}
+  @terminal.status.exitstatus.should == 0
+  # show errors
+  output_log = @terminal.output
+  is_pass = false
+  # TODO: fix regexp later, because can PASS in 10 or 100 failures
+  is_pass = true if ((1 == output_log.scan(/([.*]?)fail: 0(\D+)error: 0([\D+]?)/i).size) || (1 == output_log.scan(/(.*)0 failures(.*)0 errors(.*)/i).size))
+  puts @terminal.output unless is_pass
+  # check if have errors
+  is_pass.should == true
+end
+
+When /^I run "([^\"]*)"$/ do |command|
+  @terminal.cd(rails_root)
+  @terminal.run(command)
+end

data/features/support/env.rb

@@ -0,0 +1,15 @@
+require 'active_support'
+require 'rspec'
+
+PROJECT_ROOT     = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')).freeze
+TEMP_DIR         = File.join(PROJECT_ROOT, 'tmp').freeze
+LOCAL_RAILS_ROOT = File.join(TEMP_DIR, 'rails_root').freeze
+BUILT_GEM_ROOT   = File.join(TEMP_DIR, 'built_gems').freeze
+LOCAL_GEM_ROOT   = File.join(TEMP_DIR, 'local_gems').freeze
+
+Before do
+  FileUtils.mkdir_p(TEMP_DIR)
+  FileUtils.rm_rf(BUILT_GEM_ROOT)
+  FileUtils.rm_rf(LOCAL_RAILS_ROOT)
+  FileUtils.mkdir_p(BUILT_GEM_ROOT)
+end

data/features/support/rails.rb

@@ -0,0 +1,98 @@
+module RailsHelpers
+  def rails_root_exists?
+    File.exists?(environment_path)
+  end
+  
+  def environment_path
+    File.join(rails_root, 'config', 'environment.rb')
+  end
+
+  def application_controller_filename
+    controller_filename = File.join(rails_root, 'app', 'controllers', "application_controller.rb")
+  end
+
+  def rails_root
+    LOCAL_RAILS_ROOT
+  end
+  
+  def rails30?
+    rails_version =~ /^3.0/
+  end
+  
+  def rails31?
+    rails_version =~ /^3.1/
+  end
+
+  def rails_version
+    @rails_version ||= begin
+      rails_version = open(gemfile_path).read.match(/gem.*rails["'].*["'](.+)["']/)[1]
+    end
+  end
+
+  def gemfile_path
+    File.join(rails_root, 'Gemfile')
+  end
+
+  def rakefile_path
+    File.join(rails_root, 'Rakefile')
+  end
+  
+  def routes_path
+    File.join(rails_root, 'config', 'routes.rb')
+  end
+  
+  def logs_path
+    File.join(rails_root, 'log')
+  end
+
+  def bundle_gem(gem_name, version = nil)
+    File.open(gemfile_path, 'a') do |file|
+      gem = "gem '#{gem_name}'"
+      gem += ", '#{version}'" if version
+      file.puts(gem)
+    end
+  end
+  
+  def require_thread
+    content = File.read(rakefile_path)
+    content = "require 'thread'\n#{content}"
+    File.open(rakefile_path, 'wb') { |file| file.write(content) }
+  end
+  
+  def add_routes
+    content = File.read(routes_path)
+    flag = Regexp.escape("Application.routes.draw do\n")
+    order_routes = <<STR
+resources :order do
+  collection do
+    post :test_post
+  end
+end
+STR
+    content.gsub!(/#{flag}/m, '\0 ' + order_routes)
+    File.open(routes_path, 'wb') { |file| file.write(content) }
+  end
+  
+  def copy_tests
+    FileUtils.cp(
+      File.join(PROJECT_ROOT, 'test', 'rails', 'app', 'controllers', 'order_controller.rb'), 
+      File.join(rails_root, 'app', 'controllers', 'order_controller.rb')
+    )
+    FileUtils.cp(
+      File.join(PROJECT_ROOT, 'test', 'config', 'samples', 'database.yml'), 
+      File.join(rails_root, 'config', 'database.yml')
+    )
+    FileUtils.cp(
+      File.join(PROJECT_ROOT, 'test', 'rails', 'test', 'test_helper.rb'), 
+      File.join(rails_root, 'test', 'test_helper.rb')
+    )
+    FileUtils.cp(
+      File.join(PROJECT_ROOT, 'test', 'rails', 'test', 'functional', 'order_controller_test.rb'), 
+      File.join(rails_root, 'test', 'functional', 'order_controller_test.rb')
+    )
+    FileUtils.chmod 0777, logs_path
+  end
+  
+end
+
+World(RailsHelpers)

data/features/support/terminal.rb

@@ -0,0 +1,95 @@
+require 'fileutils'
+
+Before do
+  @terminal = Terminal.new
+end
+
+After do |story|
+  if story.failed?
+    # puts @terminal.output
+  end
+end
+
+class Terminal
+  attr_reader :output, :status
+  attr_accessor :environment_variables
+
+  def initialize
+    @cwd = FileUtils.pwd
+    @output = ""
+    @status = 0
+    @logger = Logger.new(File.join(TEMP_DIR, 'terminal.log'))
+    
+    @environment_variables = {
+      "GEM_HOME" => LOCAL_GEM_ROOT,
+      "GEM_PATH" => "#{LOCAL_GEM_ROOT}:#{BUILT_GEM_ROOT}",
+      "PATH" => "#{gem_bin_path}:#{ENV['PATH']}",
+      "BUNDLE_GEMFILE" => File.join(LOCAL_RAILS_ROOT, 'Gemfile')
+    }
+  end
+
+  def cd(directory)
+    @cwd = directory
+  end
+
+  def run(command)
+    output << "#{command}\n"
+    current_dir = FileUtils.pwd
+    
+    # The ; forces ruby to shell out so the env settings work right
+    cmdline = "sh -c 'cd #{@cwd} && #{environment_settings} #{command} 2>&1 ; '"
+    logger.debug(cmdline)
+    result = `#{cmdline}`
+    logger.debug(result)
+    output << result
+    
+    `sh -c 'cd #{current_dir}'`
+    @status = $?
+  end
+
+  def echo(string)
+    logger.debug(string)
+  end
+
+
+  def build_and_install_gem(gemspec)
+    pkg_dir = File.join(TEMP_DIR, 'pkg')
+    FileUtils.mkdir_p(pkg_dir)
+    output = `gem build #{gemspec} 2>&1`
+    gem_file = Dir.glob("*.gem").first
+    unless gem_file
+      raise "Gem didn't build:\n#{output}"
+    end
+    target = File.join(pkg_dir, gem_file)
+    FileUtils.mv(gem_file, target)
+    install_gem_to(LOCAL_GEM_ROOT, target)
+  end
+
+  def install_gem(gem)
+    install_gem_to(LOCAL_GEM_ROOT, gem)
+  end
+
+  def uninstall_gem(gem)
+    `gem uninstall -i #{LOCAL_GEM_ROOT} #{gem}`
+  end
+
+  def prepend_path(path)
+    @environment_variables['PATH'] = path + ":" + @environment_variables['PATH']
+  end
+
+  private
+
+  def install_gem_to(root, gem)
+    `gem install -i #{root} --no-ri --no-rdoc #{gem}`
+  end
+
+  def environment_settings
+    @environment_variables.map { |key, value| "#{key}=#{value}" }.join(' ')
+  end
+
+  def gem_bin_path
+    File.join(LOCAL_GEM_ROOT, "bin")
+  end
+
+  attr_reader :logger
+end

data/lib/mongodb_logger/initializer_mixin.rb

@@ -0,0 +1,26 @@
+module MongodbLogger
+  module InitializerMixin
+    
+    def rails30?
+      3 == Rails::VERSION::MAJOR && 0 == Rails::VERSION::MINOR
+    end
+    
+    def create_logger(config)
+      path = rails30? ? config.paths.log.to_a.first : config.paths['log'].first
+      level = ActiveSupport::BufferedLogger.const_get(config.log_level.to_s.upcase)
+      logger = MongodbLogger::Logger.new(:path => path, :level => level)
+      logger.auto_flushing = false if Rails.env.production?
+      logger
+    rescue StandardError => e
+      logger = ActiveSupport::BufferedLogger.new(STDERR)
+      logger.level = ActiveSupport::BufferedLogger::WARN
+      logger.warn(
+        "MongodbLogger Initializer Error: Unable to access log file. Please ensure that #{path} exists and is chmod 0666. " +
+        "The log level has been raised to WARN and the output directed to STDERR until the problem is fixed." + "\n" +
+        e.message + "\n" + e.backtrace.join("\n")
+      )
+      logger
+    end
+    
+  end
+end

data/lib/mongodb_logger/logger.rb

@@ -0,0 +1,239 @@
+require 'erb'
+require 'mongo'
+require 'active_support'
+require 'active_support/core_ext'
+require 'action_dispatch/http/upload'
+require 'mongodb_logger/replica_set_helper'
+
+module MongodbLogger
+  class Logger < ActiveSupport::BufferedLogger
+    include ReplicaSetHelper
+
+    DEFAULT_COLLECTION_SIZE = 250.megabytes
+    # Looks for configuration files in this order
+    CONFIGURATION_FILES = ["mongodb_logger.yml", "mongoid.yml", "database.yml"]
+    LOG_LEVEL_SYM = [:debug, :info, :warn, :error, :fatal, :unknown]
+
+    attr_reader :db_configuration, :mongo_connection, :mongo_collection_name, :mongo_collection, :mongo_connection_type
+
+    def initialize(options={})
+      path = options[:path] || File.join(Rails.root, "log/#{Rails.env}.log")
+      level = options[:level] || DEBUG
+      internal_initialize
+    rescue => e
+      # should use a config block for this
+      Rails.env.production? ? (raise e) : (puts "MongodbLogger WARNING: Using BufferedLogger due to exception: " + e.message)
+    ensure
+      if disable_file_logging?
+        @level          = level
+        @buffer         = {}
+        @auto_flushing  = 1
+        @guard          = Mutex.new
+      else
+        super(path, level)
+      end
+    end
+
+    def add_metadata(options={})
+      options.each do |key, value|
+        unless [:messages, :request_time, :ip, :runtime, :application_name, :is_exception, :params, :method].include?(key.to_sym)
+          @mongo_record[key] = value
+        else
+          raise ArgumentError, ":#{key} is a reserved key for the mongodb logger. Please choose a different key"
+        end
+      end
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      if @level && @level <= severity && message.present? && @mongo_record.present?
+        # do not modify the original message used by the buffered logger
+        msg = logging_colorized? ? message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, '').strip : message
+        @mongo_record[:messages][LOG_LEVEL_SYM[severity]] << msg
+      end
+      # may modify the original message
+      disable_file_logging? ? message : (@level ? super : message)
+    end
+
+    # Drop the capped_collection and recreate it
+    def reset_collection
+      if @mongo_connection && @mongo_collection
+        @mongo_collection.drop
+        create_collection
+      end
+    end
+
+    def mongoize(options={})
+      @mongo_record = options.merge({
+        :messages => Hash.new { |hash, key| hash[key] = Array.new },
+        :request_time => Time.now.getutc,
+        :application_name => @application_name
+      })
+
+      runtime = Benchmark.measure{ yield }.real if block_given?
+    rescue Exception => e
+      add(3, e.message + "\n" + e.backtrace.join("\n"))
+      # log exceptions
+      @mongo_record[:is_exception] = true
+      # Reraise the exception for anyone else who cares
+      raise e
+    ensure
+      # In case of exception, make sure runtime is set
+      @mongo_record[:runtime] = ((runtime ||= 0) * 1000).ceil
+      begin
+        @insert_block.call
+      rescue
+        begin
+          # try to nice serialize record
+          nice_serialize @mongo_record
+          @insert_block.call
+        rescue
+          # do extra work to inpect (and flatten)
+          force_serialize @mongo_record
+          @insert_block.call rescue nil
+        end
+      end
+    end
+
+    def authenticated?
+      @authenticated
+    end
+
+    private
+      # facilitate testing
+      def internal_initialize
+        configure
+        connect
+        check_for_collection
+      end
+      
+      def disable_file_logging?
+        @db_configuration.fetch('disable_file_logging', false)
+      end
+
+      def configure
+        default_capsize = DEFAULT_COLLECTION_SIZE
+        @authenticated = false
+        @db_configuration = {
+          'host' => 'localhost',
+          'port' => 27017,
+          'capsize' => default_capsize}.merge(resolve_config)
+        @mongo_collection_name = @db_configuration['collection'] || "#{Rails.env}_log"
+        @application_name = resolve_application_name
+        @safe_insert = @db_configuration['safe_insert'] || false
+
+        @insert_block = @db_configuration.has_key?('replica_set') && @db_configuration['replica_set'] ?
+          lambda { rescue_connection_failure{ insert_log_record(@safe_insert) } } :
+          lambda { insert_log_record }
+      end
+
+      def resolve_application_name
+        if @db_configuration.has_key?('application_name')
+          @db_configuration['application_name']
+        else
+          Rails.application.class.to_s.split("::").first
+        end
+      end
+
+      def resolve_config
+        config = {}
+        CONFIGURATION_FILES.each do |filename|
+          config_file = Rails.root.join("config", filename)
+          if config_file.file?
+            config = YAML.load(ERB.new(config_file.read).result)[Rails.env]
+            config = config['mongodb_logger'] if config && config.has_key?('mongodb_logger')
+            break unless config.blank?
+          end
+        end
+        config
+      end
+
+      def mongo_connection_object
+        if @db_configuration['hosts']
+          conn = Mongo::ReplSetConnection.new(*(@db_configuration['hosts'] << 
+            {:auto_reconnect => true, :pool_timeout => 6}))
+          @db_configuration['replica_set'] = true
+        else
+          conn = Mongo::Connection.new(@db_configuration['host'],
+                                       @db_configuration['port'],
+                                       :auto_reconnect => true,
+                                       :pool_timeout => 6)
+        end
+        @mongo_connection_type = conn.class
+        conn
+      end
+
+      def connect
+        @mongo_connection ||= mongo_connection_object.db(@db_configuration['database'])
+        if @db_configuration['username'] && @db_configuration['password']
+          # the driver stores credentials in case reconnection is required
+          @authenticated = @mongo_connection.authenticate(@db_configuration['username'],
+                                                          @db_configuration['password'])
+        end
+      end
+
+      def create_collection
+        @mongo_connection.create_collection(@mongo_collection_name,
+                                            {:capped => true, :size => @db_configuration['capsize'].to_i})
+      end
+
+      def check_for_collection
+        # setup the capped collection if it doesn't already exist
+        create_collection unless @mongo_connection.collection_names.include?(@mongo_collection_name)
+        @mongo_collection = @mongo_connection[@mongo_collection_name]
+      end
+
+      def insert_log_record(safe = false)
+        @mongo_collection.insert(@mongo_record, :safe => safe)
+      end
+
+      def logging_colorized?
+        # Cache it since these ActiveRecord attributes are assigned after logger initialization occurs in Rails boot
+        @colorized ||= Object.const_defined?(:ActiveRecord) && ActiveRecord::LogSubscriber.colorize_logging
+      end
+      
+      # try to serialyze data by each key and found invalid object
+      def nice_serialize(rec)
+        if msgs = rec[:messages]
+          msgs.each do |i, j|
+            msgs[i] = nice_serialize_object(j)
+          end
+        end
+        if pms = rec[:params]
+          pms.each do |i, j|
+            pms[i] = nice_serialize_object(j)
+          end
+        end
+      end
+      
+      def nice_serialize_object(data)
+        case data
+          when NilClass, String, Fixnum, Bignum, Float, TrueClass, FalseClass, Time, Regexp, Symbol
+            data
+          when Hash
+            hvalues = Hash.new
+            data.each{|k,v| hvalues[k] = nice_serialize_object(v) }
+            hvalues
+          when Array
+            data.map{|v| nice_serialize_object(v) }
+          when ActionDispatch::Http::UploadedFile, Rack::Test::UploadedFile # uploaded files
+            hvalues = {
+              :original_filename => data.original_filename,
+              :content_type => data.content_type
+            }
+          else
+            data.inspect
+        end
+      end
+
+      # force the data in the db by inspecting each top level array and hash element
+      # this will flatten other hashes and arrays
+      def force_serialize(rec)
+        if msgs = rec[:messages]
+          msgs.each { |i, j| msgs[i] = j.inspect }
+        end
+        if pms = rec[:params]
+          pms.each { |i, j| pms[i] = j.inspect }
+        end
+      end
+  end
+end

data/lib/mongodb_logger/railtie.rb

@@ -0,0 +1,12 @@
+require 'mongodb_logger/initializer_mixin'
+module MongodbLogger
+  class Railtie < Rails::Railtie
+    include MongodbLogger::InitializerMixin
+    
+    initializer :initialize_mongodb_logger, :before => :initialize_logger do
+      app_config = Rails.application.config
+      Rails.logger = config.logger = create_logger(app_config)
+    end
+    
+  end
+end

data/lib/mongodb_logger/replica_set_helper.rb

@@ -0,0 +1,19 @@
+module MongodbLogger
+  module ReplicaSetHelper
+    # Use retry alg from mongodb to gobble up connection failures during replica set master vote
+    # Defaults to a 10 second wait
+    def rescue_connection_failure(max_retries=40)
+      success = false
+      retries = 0
+      while !success
+        begin
+          yield
+          success = true
+        rescue Mongo::ConnectionFailure => e
+          raise e if (retries += 1) >= max_retries
+          sleep 0.25
+        end
+      end
+    end
+  end
+end