mongo_request_logger 0.1.0

data/lib/mongo_request_logger/log_message.rb

@@ -0,0 +1,170 @@
+require 'net/http'
+require 'rack/utils'
+
+module MongoRequestLogger
+  class LogMessage
+    attr_reader :data
+
+    def initialize data={}
+      @data = data
+    end
+
+    def [](param)
+      @data[param.to_s]
+    end
+
+    def id
+      self[:_id].to_s
+    end
+
+    def referer
+      referer = self[:referer]
+      referer_data = {
+        'annotation' => ''
+      }
+
+      # parse google adwords HTTP_REFERERs for actual referer page on display network
+      if referer.nil? or referer.empty?
+        referer_data['url'] = ''
+
+      elsif referer.include? 'http://googleads.g.doubleclick.net'
+        uri = URI(referer)
+        params = ::Rack::Utils.parse_nested_query(uri.query)
+        referer_data['url'] = params['url']
+        referer_data['annotation'] = 'AdWords Display Network'
+
+      # parse google adwords SERP clicks for actual SERP page
+      elsif referer.include?('google') and referer.include?('aclk') and referer.include?('adurl')
+        uri = URI(referer)
+        params = ::Rack::Utils.parse_nested_query(uri.query)
+        referer_data['url'] = "http://#{uri.host}/?q=#{CGI::escape(params['q'])}"
+        referer_data['annotation'] = 'Adwords Google Search'
+
+      else
+        referer_data['url'] = referer
+      end
+      referer_data
+    end
+
+    def path
+      self[:path]
+    end
+
+    def runtime
+      "%.3f" % (self[:runtime] / 1000.0) if self[:runtime]
+    end
+
+    # One of:
+    # ''
+    # 'user_id'
+    # 'user_name'
+    # 'user_id - user_name'
+    def user
+      {
+        name: (self[:user_name] or ''),
+        email: (self[:user_email] or ''),
+        email_md5: Digest::MD5.hexdigest((self[:user_email] or ''))
+      }
+    end
+
+    def user_id
+      self[:user_id]
+    end
+
+    def user_agent
+      self[:user_agent]
+    end
+
+    def time
+      self[:timestamp].getlocal.strftime("%y-%m-%d %H:%M:%S") if self[:timestamp]
+    end
+
+    def messages
+      self[:messages]
+    end
+
+    def response
+      response_code = self[:response]
+      if self[:exception]
+        response_description = self[:exception]['class']
+      else
+        response_class = Net::HTTPResponse::CODE_TO_OBJ[self[:response].to_s]
+        if response_class
+          response_description = response_class.name.split("::HTTP").last
+        else
+          response_description = 'OK'
+        end
+      end
+
+      [response_code, response_description].select {|v| v != nil}.join(" - ")
+    end
+
+    def exception
+      if self[:exception]
+        if self[:exception]['code']
+          "#{self[:exception]['class']} - [#{self[:exception]['code']}] #{self[:exception]['message']}" if self[:exception]
+        else
+          "#{self[:exception]['class']} - #{self[:exception]['message']}" if self[:exception]
+        end
+      end
+
+    end
+
+    def args
+      if self[:args]
+        self[:args].map { |arg| arg.to_s.truncate(50) }
+      end
+    end
+
+    def params
+      if self[:params].is_a? Array
+        result = []
+        self[:params].each do |key, value|
+          result << "#{key}: #{value.inspect.truncate(100)}"
+        end
+        result.join(", ")
+      elsif self[:params].is_a? String
+        self[:params]
+      end
+    end
+
+    def timestamp
+      (self[:timestamp].to_f * 1000).to_i
+    end
+
+    def ip
+      self[:ip] || ''
+    end
+
+    def tags
+      (self[:tags] || []) - %w(rack)
+    end
+
+    def as_json(*)
+      {
+          id: id,
+          time: time,
+          path: path,
+          referer: referer,
+          tags: tags,
+          user: user,
+          ip: ip,
+          response: response,
+          user_agent: user_agent,
+          runtime: runtime,
+          timestamp: timestamp,
+      }
+    end
+
+
+    def extra exclude=[]
+      result = {}
+      @data.each do |key, value|
+        next if exclude.include? key.to_s
+        next if value.blank?
+        result[key.to_s.titlecase] = value
+      end
+      result
+    end
+  end
+end

data/lib/mongo_request_logger/logged_job.rb

@@ -0,0 +1,63 @@
+require 'mongo_request_logger/logger'
+require 'mongo_request_logger/rack'
+
+module MongoRequestLogger
+  # Usage:
+  #
+  #   class MyJob
+  #     extend MongoRequestLogger::LoggedJob
+  #
+  #     def perform(args)
+  #       ...
+  #     end
+  #   end
+  module LoggedJob
+    def self.logger= logger
+      @logger = logger
+    end
+
+
+    def self.logger
+      @logger ||= ::MongoRequestLogger::Rack.logger
+    end
+
+    def logger
+      LoggedJob.logger
+    end
+
+    def around_perform_log_job(*args, &block)
+      call_logged *args, &block
+    end
+
+    private
+
+    def call_logged *args, &block
+      unless logger.respond_to?(:log_request)
+        # No MongoDB logger, but still want to log any errors
+        begin
+          return block.call(*args)
+        rescue Exception => e
+          logger.error e.to_s
+          logger.error e.backtrace.join("\n")
+          raise
+        end
+      end
+
+      # Note that @queue is the default queue for the job class, and may be different from the queue it was on.
+      # Therefore we don't log it anymore, to prevent confusion (until we find a way to determine that actual queue that
+      # the job was on).
+
+      options = {
+          :type => "job",
+          :job => self.name,
+          :args => args,
+      }
+      logger.log_request(options) do
+        logger.tag "resque"
+        short_name = self.name.split('::').last.underscore
+        logger.add_metadata path: "resque/#{short_name}"
+        block.call(*args)
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/logger.rb

@@ -0,0 +1,122 @@
+require 'active_support/buffered_logger'
+require 'mongo_request_logger/ext/buffered_logger'
+
+module MongoRequestLogger
+  class Logger < ActiveSupport::BufferedLogger
+    attr_reader :adapter
+
+    def initialize(adapter, file, level=DEBUG)
+      super(file, level)
+      @adapter = adapter
+      @mongo_record = {}  # This is required as a workaround for logging in the console
+      create_indexes
+    end
+
+    def create_indexes
+      adapter.create_index '_id'
+      adapter.create_index 'objects'
+      adapter.create_index 'tags'
+      adapter.create_index 'path'
+      adapter.create_index 'user_id'
+      adapter.create_index 'timestamp'
+    end
+
+    LEVELS = {
+        DEBUG => 'DEBUG',
+        INFO => 'INFO',
+        WARN => 'WARN',
+        ERROR => 'ERROR',
+        FATAL => 'FATAL',
+        UNKNOWN => 'UNKNOWN',
+    }
+
+    def level_to_s(level)
+      LEVELS[level]
+    end
+
+    def sanitize obj
+      if obj.is_a? Hash
+        result = {}
+        obj.each do |k, v|
+          # We replace all non-alphanumeric characters with underscores
+          result[k.to_s.gsub(/\W/, "_")] = sanitize(v)
+        end
+        result
+      elsif obj.is_a? Array or obj.is_a? Set
+        obj.map {|o| sanitize(o)}
+      elsif obj.is_a? Time or obj.is_a? String or obj.is_a? Integer
+        obj
+      else
+        obj.to_s
+      end
+    end
+
+    def log_request(options={})
+      @mongo_record = options.merge({
+                                        :messages => [],
+                                        :timestamp => Time.now.utc,
+                                        :pid => Process.pid,
+                                    })
+
+      begin
+        start_time = Time.now
+        return yield
+      rescue Exception => e
+        # The backtrace is not useful metadata, and we want to write it to the text log as well.
+        exception e
+        @mongo_record[:exception] = {:class => e.class.to_s, :message => e.message}
+        raise
+      ensure
+        # Benchmark.measure doesn't handle exceptions well, so we measure the time ourselves.
+        end_time = Time.now
+        runtime = end_time.to_f - start_time.to_f
+        @mongo_record[:runtime] = (runtime.real * 1000).ceil
+
+        do_log @mongo_record
+      end
+    end
+
+    def add_metadata(options={})
+      options.each_pair do |key, value|
+        @mongo_record[key] = value
+      end
+    end
+
+    def add_metadata_set key, *values
+      key = key.to_s
+      @mongo_record[key] ||= Set.new
+      @mongo_record[key] += values
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      unless level > severity
+        time = Time.now.strftime "%H:%M:%S.%L"
+        prefix = "#{time} [#{level_to_s(severity)}] "
+        #if ActiveRecord::Base.colorize_logging
+        #  # remove colorization done by rails and just save the actual message
+        #  @mongo_record[:messages] << prefix + message.gsub(/(\e(\[([\d;]*[mz]?))?)?/, '').strip rescue nil
+        #else
+        @mongo_record[:messages] ||= []
+        @mongo_record[:messages] << prefix + message
+        #end
+      end
+
+      # To do normal file logging as well, add a call to super here
+      # We only log ERROR and above to the text file logs.
+      if severity >= ERROR
+        super
+      end
+    end
+
+    private
+    def do_log(record)
+      begin
+        adapter.insert_log_record(sanitize(record))
+      rescue => e
+        # Note that this will not result in recursion, as we do not commit the log again here. It will be written to the
+        # text log.
+        exception e, "Unable to log to MongoDB"
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/logger_extensions.rb

@@ -0,0 +1,34 @@
+require 'mongo_request_logger/backtrace_cleaner'
+
+# Some additional logging methods. These are as extensions to the standard logger classes instead of our logger, so
+# that an app can easily switch between the structured logger and a standard file-based logger.
+module MongoRequestLogger
+  module LoggerExtensions
+    def add_metadata_set(key, *values)
+      info "#{key}: #{values.inspect}"
+    end
+
+    def add_metadata(options={})
+      info options.inspect
+    end
+
+    def tag *tags
+      add_metadata_set :tags, *tags
+    end
+
+    def exception(exception, message=nil)
+      if message
+        start = "#{message}: "
+      else
+        start = ""
+      end
+
+      cleaned_backtrace = MongoRequestLogger::BacktraceCleaner.clean(exception.backtrace || [])
+
+      log_message = "#{start}#{exception.class} #{exception.message}:\n  " +
+          cleaned_backtrace.join("\n  ")
+
+      error log_message
+    end
+  end
+end

data/lib/mongo_request_logger/rack.rb

@@ -0,0 +1,63 @@
+require 'logger'
+require 'rack/request'
+require 'active_support/core_ext/class/attribute_accessors'
+
+
+module MongoRequestLogger
+  class Rack
+    cattr_accessor :logger
+    cattr_accessor :ignore_prefixes
+
+    def self.ignore_prefix prefix
+      self.ignore_prefixes ||= []
+      self.ignore_prefixes << prefix
+    end
+
+    self.ignore_prefix '/log'   # TODO: dynamically detect this?
+
+    def initialize(app, options = {})
+      @app = app
+      @options = options
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    def call(env)
+      #HACK: Don't log the log viewer itself
+      self.class.ignore_prefixes.each do |prefix|
+        if env["PATH_INFO"].to_s.start_with? prefix
+          return @app.call(env)
+        end
+      end
+      # TODO: filter parameters
+      # KLUDGE: this will cause parameter parsing to happen twice: once here, once later on in Rails.
+      # Or maybe it's automatically cached in env?
+      logger.log_request do
+        begin
+          request = ::Rack::Request.new(env)
+          options = {
+              path: request.path_info,
+              host: request.host_with_port,
+              user_agent: request.user_agent,
+              ip: request.ip,
+              referer: request.referrer,
+              query_string: request.query_string,
+              request_method: request.request_method,
+              content_type: request.content_type,
+              params: request.params
+          }
+          logger.add_metadata options
+          logger.add_metadata_set :tags, "rack"
+          status, headers, body = @app.call(env)
+          logger.add_metadata response: status.to_i
+          [status, headers, body]
+        rescue Exception => e
+          logger.add_metadata response: 500
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/railtie.rb

@@ -0,0 +1,57 @@
+if (defined? Rails) && Rails.version =~ /^3\./
+  require 'mongo_request_logger'
+  require 'mongo_request_logger/config'
+  require 'mongo_request_logger/rack'
+  require 'mongo_request_logger/viewer'
+
+  require 'rails'
+
+  begin
+    # We prefer moped ...
+    require 'moped'
+  rescue LoadError
+    # ... but fallback to mongo
+    require 'mongo'
+  end
+
+  module MongoRequestLogger
+    class Railtie < ::Rails::Railtie
+      class << self
+        def setup(app)
+
+          global_config = MongoRequestLogger::Config.new
+          global_config.load_file(File.join(Rails.root, "config/logger.yml"))
+          log_config = global_config.namespaced(Rails.env)
+          return unless log_config && log_config['database']
+
+          if defined? Moped
+            adapter = MongoRequestLogger::Adapters::Moped.new(log_config)
+          else
+            adapter = MongoRequestLogger::Adapters::Mongo.new(log_config)
+          end
+          Rails.logger = MongoRequestLogger::Logger.new adapter, Rails.root.join("log/#{Rails.env}.log")
+
+          MongoRequestLogger::Rack.logger = Rails.logger
+          MongoRequestLogger::Rack.ignore_prefixes << '/assets'
+
+          MongoRequestLogger::Viewer.adapter = adapter
+
+          app.config.middleware.insert_after ActionDispatch::DebugExceptions, MongoRequestLogger::Rack
+
+          if defined?(PhusionPassenger)
+            PhusionPassenger.on_event(:starting_worker_process) do |forked|
+              if forked
+                adapter.reconnect
+              end
+            end
+          end
+
+        end
+      end
+
+      initializer "setup logger" do |app|
+        MongoRequestLogger::Railtie.setup(app)
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/search_terms.rb

@@ -0,0 +1,59 @@
+# Used the following blog post as a starting point, then completely rewrote the parser:
+# http://proccli.com/2011/12/advanced-search-query-parsing-ruby/
+# https://gist.github.com/1477730#file_search_terms.rb
+
+module MongoRequestLogger
+  class SearchTerms
+    attr_reader :query, :parts, :search, :path, :search_regex
+
+    # query:: this is what you want tokenized
+    # split:: if you'd like to split values on "," then pass true
+    def initialize(query)
+      @query = query
+      @parts = {}
+      @search = nil
+      @search_regex = nil
+      parse_query
+    end
+
+    def [](key)
+      @parts[key]
+    end
+
+    private
+
+    SCANNER = /(?:(\w+):(\S+))  # key:value
+              |(?:"(.+)")       # string in quotes
+              |(^\S+)           # a path
+              |(?:\/(.+?)\/(?:\s|$))  # regular expression
+              |(\S+)/x          # a word
+    def parse_query
+      search_parts = []
+      search_regex = nil
+      # We break up the query into a sequence of the following possible components:
+      #   key:value
+      #   "a string wrapped in quotes"
+      #   /a/path (at the beginning of the string only)
+      #   /a regular expression/
+      #   word
+      groups = @query.scan(SCANNER)
+      groups.map do |key, value, search, path, regex, word|
+        if key
+          @parts[key.downcase] = value
+        elsif search
+          search_parts << search
+        elsif regex
+          @search_regex = regex
+        elsif path
+          @path = path
+        elsif word
+          search_parts << word
+        end
+      end
+
+      unless @search_regex
+        @search = search_parts.join(' ')
+      end
+    end
+  end
+end

data/lib/mongo_request_logger/version.rb

@@ -0,0 +1,3 @@
+module MongoRequestLogger
+  VERSION = '0.1.0'
+end