superlogger 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a8233a0f1140b23ad1a79d954256e9f406e7dbf4
-  data.tar.gz: 7fc49063ae14c89e033a224dc619e878401d8f33
+SHA256:
+  metadata.gz: 6c620e677e4aebf4aea5cab58f2a6e392c5266016eaf32aef3a9a88f96c402fd
+  data.tar.gz: 59fd2728e8398194e9188f212d7dc3f1155f1c2f101d0916747fbcbd52d1c23c
 SHA512:
-  metadata.gz: 5bf2ca0272093a2b54928be31590756f57ddb0fb959c0fc4fc37cdd074cdaf51adbc903bfd5d549705708b53269e312afeb20ac04af50dec472ff41e7c4082c8
-  data.tar.gz: 1b181052bfb35cc721824dde111d9d87b4aadc0b8605d944b7c1657e0afff478a1b2052c6033b8df04f64f262de53b84744c709c177bed20985f9799ae8cafef
+  metadata.gz: fd1ab3fe21cd6759e6732746fd6e206a71816f9926559098a0815d6940aa058568c9f5204b48dd643fe74b1639ddbe938dc11f9817ce5485058ade00207b451d
+  data.tar.gz: b55db9a3bccd0ce12fb20666b40f85d73c43b873ff9c9dab7cc776b3cd4667a7939202a18ab41212f9c33c31f9b5bebfde84a994b2126bf0a58ba8d13528fb5c

data/README.md

@@ -1,82 +1,74 @@
-Superlogger - Machine-readable logging for Rails [![Build Status](https://travis-ci.org/moexmen/superlogger.svg?branch=master)](https://travis-ci.org/moexmen/superlogger)
+[![Build Status](https://travis-ci.org/moexmen/superlogger.svg?branch=master)](https://travis-ci.org/moexmen/superlogger)
+[![Gem Version](https://badge.fury.io/rb/superlogger.svg)](https://badge.fury.io/rb/superlogger)
+[![Dependency Status](https://gemnasium.com/badges/github.com/moexmen/superlogger.svg)](https://gemnasium.com/github.com/moexmen/superlogger)
+
+Superlogger - Machine-readable logging for Rails
 =======
 
-Rails' default request logging is easy to read for humans but difficult for log aggregators such as Kibana, Graylog and Splunk. Superlogger transforms the logs into key-value pairs for easy parsing and adds useful details like Timestamp and Session ID for tracing purposes.
+Rails' default request logging is easy to read for humans but difficult for log aggregators such as Kibana, Graylog and Splunk. Superlogger transforms the logs into JSON for easy parsing and adds useful details like Timestamp, Session ID and Request ID for tracing purposes.
 
 Default rails logging:
 ```sh
-Started GET "/home/index" for ::1 at 2016-04-11 16:23:27 +0800
+Started GET "/home/index" for ::1 at 2016-04-29 17:31:12 +0800
   ActiveRecord::SchemaMigration Load (0.1ms)  SELECT "schema_migrations".* FROM "schema_migrations"
 Processing by HomeController#index as HTML
-  Something Load (0.2ms)  SELECT  "somethings".* FROM "somethings" WHERE "somethings"."paper" = ? AND "somethings"."stone" = ?  ORDER BY "somethings"."id" ASC LIMIT 1  [["paper", "123"], ["stone", "456"]]
+  Something Load (0.1ms)  SELECT  "somethings".* FROM "somethings" WHERE "somethings"."paper" = ? AND "somethings"."stone" = ?  ORDER BY "somethings"."id" ASC LIMIT 1  [["paper", "123"], ["stone", "456"]]
   Rendered home/_partial.html.erb (0.2ms)
-  Rendered home/index.html.erb within layouts/application (5.2ms)
-Completed 200 OK in 283ms (Views: 271.6ms | ActiveRecord: 0.6ms)
+  Rendered home/index.html.erb within layouts/application (3.4ms)
+Completed 200 OK in 135ms (Views: 130.7ms | ActiveRecord: 0.3ms)
+
+Started GET "/assets/application.self-e80e8f2318043e8af94dddc2adad5a4f09739a8ebb323b3ab31cd71d45fd9113.css?body=1" for ::1 at 2016-04-29 17:31:12 +0800
 
-Started GET "/assets/application.self-e80e8f2318043e8af94dddc2adad5a4f09739a8ebb323b3ab31cd71d45fd9113.css?body=1" for ::1 at 2016-04-11 16:23:27 +0800
+Started GET "/assets/application.self-8f06a73c35179188914ab50e057157639fce1401c1cdca640ac9cec33746fc5b.js?body=1" for ::1 at 2016-04-29 17:31:12 +0800
 
-Started GET "/assets/application.self-8f06a73c35179188914ab50e057157639fce1401c1cdca640ac9cec33746fc5b.js?body=1" for ::1 at 2016-04-11 16:23:27 +0800
 ```
 
-Machine-readable logging with Superlogger:
+With Superlogger:
 ```sh
-2016-04-11 16:18:22.279 | b48534ea049a | I | middleware:28 | method=GET | path=/home/index | ip=::1
-2016-04-11 16:18:22.293 | b48534ea049a | D | action_controller_log_subscriber:9 | controller=HomeController | action=index | params={}
-2016-04-11 16:18:22.305 | b48534ea049a | D | active_record_log_subscriber:24 | sql=SELECT  "somethings".* FROM "somethings" WHERE "somethings"."paper" = ? AND "somethings"."stone" = ?  ORDER BY "somethings"."id" ASC LIMIT 1 | params=["'123'", "'456'"] | duration=0.48
-2016-04-11 16:18:22.316 | b48534ea049a | D | action_view_log_subscriber:6 | view=_partial.html.erb | duration=0.3
-2016-04-11 16:18:22.316 | b48534ea049a | D | action_view_log_subscriber:6 | view=index.html.erb | duration=5.6
-2016-04-11 16:18:22.541 | b48534ea049a | I | action_controller_log_subscriber:29 | status=200 | total_duration=247.16 | view_duration=235.25 | db_duration=0.78
+{"level":"debug","ts":1590972589.522784,"caller":"superlogger/active_record_log_subscriber:21","sql":"SELECT \"schema_migrations\".\"version\" FROM \"schema_migrations\" ORDER BY \"schema_migrations\".\"version\" ASC","params":[],"duration":0.13}
+{"level":"info","ts":1590972589.526133,"caller":"superlogger/superlogger_middleware:21","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","method":"GET","path":"/home/index"}
+{"level":"debug","ts":1590972589.546272,"caller":"superlogger/action_controller_log_subscriber:8","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","controller":"HomeController","action":"index","params":{}}
+{"level":"debug","ts":1590972589.55092,"caller":"superlogger/active_record_log_subscriber:21","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","sql":"SELECT  \"somethings\".* FROM \"somethings\" WHERE \"somethings\".\"paper\" = ? AND \"somethings\".\"stone\" = ? ORDER BY \"somethings\".\"id\" ASC LIMIT ?","params":["123","456","1"],"duration":0.22}
+{"level":"debug","ts":1590972589.574199,"caller":"superlogger/action_view_log_subscriber:6","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view":"_partial.html.erb","duration":0.33}
+{"level":"debug","ts":1590972589.574795,"caller":"superlogger/action_view_log_subscriber:6","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view":"index.html.erb","duration":2.95}
+{"level":"info","ts":1590972589.61165,"caller":"superlogger/action_controller_log_subscriber:20","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view_duration":54.59,"db_duration":0.85}
+{"level":"info","ts":1590972589.611928,"caller":"superlogger/superlogger_middleware:30","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","method":"GET","path":"/home/index","response_time":85.65,"status":200}
 ```
 
-## Features ##
-- Timestamp (milliseconds)
-- Session ID for logs across requests
-- Key-value pairs for log data
-- Muting of assets logging
-- File and line numbers
-- Recording of IP address
-
 ## Installation ##
 
 Add superlogger to your application's Gemfile
-
 ```ruby
 gem "superlogger"
 ```
 
-And then execute:
-
+Execute:
 ```sh
 $ bundle
 ```
 
-## Usage ##
+And add the following in `config/environment/production.rb`
+```ruby
+config.logger = Superlogger::Logger.new(STDOUT)
+```
 
+By default, Superlogger is only enabled in production environment because JSON is easy for machines to parse but difficult for humans to read. To forcefully enable Superlogger in non-production environment, set in `config/application.rb`:
 ```ruby
-require 'superlogger'
-
-class SomeClass
-    include Superlogger
-
-    def some_method
-        Logger.debug name:'john', age: '21'
-        Logger.info name:'john', age: '21'
-        Logger.warn name:'john', age: '21'
-        Logger.error name:'john', age: '21'
-        Logger.fatal name:'john', age: '21'
-    end
-end
+Superlogger.enabled = true
 ```
 
-## Output Format ##
-```sh
-2015-03-26 23:37:38.086 | 970298669a40   | I            | log_subscriber:24           | status:200 | total:858.35 | view:597.46 | db:34.96
-< timestamp >           | < session id > | < severity > | < file >:< line num >       | < data you pass in ... >
+## Usage ##
+
+Log as per normal using `Rails.logger`.
+
+```ruby
+Rails.logger.info foo:'true', bar: 'false'
+Rails.logger.info "Meatball"
 ```
 
-### Severity Levels ###
-- **D** - Debug
-- **I** - Info
-- **W** - Warn
-- **E** - Error
-- **F** - Fatal
+## Log Format ##
+- `ts` = Unix Epoch timestamp
+- `session_id` = Truncated to 12 characters
+- `request_id` = 32 characters
+- `msg` = If values given is not a hash, it is treated as `{"msg":<value>"}`
+- All duration related fields are in milliseconds

data/lib/superlogger.rb

@@ -2,46 +2,47 @@ require 'superlogger/version'
 require 'superlogger/logger'
 
 module Superlogger
+  @@enabled = false
+
   module_function
 
   def setup(app)
-    overwrite_rails_rack_logger
-    overwrite_action_dispatch_debug_exceptions
+    return unless (Rails.env.production? || enabled)
+    
     insert_superlogger_middleware(app)
-    detach_all_existing_log_subscribers
+    detach_existing_log_subscribers
     attach_superlogger_log_subscribers
   end
 
-  def overwrite_rails_rack_logger
-    require 'superlogger/rails_rack_logger'
-  end
-
-  def overwrite_action_dispatch_debug_exceptions
-    require 'superlogger/action_dispatch_debug_exceptions'
-  end
-
   def insert_superlogger_middleware(app)
-    require 'superlogger/middleware'
+    require 'superlogger/superlogger_middleware'
 
     # important to insert after session middleware so we can get the session id
-    app.middleware.use Superlogger::Middleware
+    app.middleware.use Superlogger::SuperloggerMiddleware
   end
 
-  def detach_all_existing_log_subscribers
+  def detach_existing_log_subscribers
     # force log subscribers to attach first so we can remove them all
     require 'action_controller/log_subscriber'
     require 'active_record/log_subscriber'
     require 'action_view/log_subscriber'
-    require 'action_mailer/log_subscriber'
 
     # remove log subscribers
+    remove_patterns = %w(sql.active_record
+                  start_processing.action_controller
+                  process_action.action_controller
+                  render_template.action_view
+                  render_partial.action_view
+                  render_collection.action_view)
+
     ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
-      subscriber.patterns.each do |pattern|
-        ActiveSupport::Notifications.unsubscribe pattern
+      patterns = subscriber.patterns
+      patterns = patterns.is_a?(Hash) ? patterns.keys : patterns
+
+      patterns.each do |pattern|
+        ActiveSupport::Notifications.unsubscribe pattern if remove_patterns.include?(pattern)
       end
     end
-
-    ActiveSupport::LogSubscriber.log_subscribers.clear
   end
 
   def attach_superlogger_log_subscribers
@@ -49,6 +50,30 @@ module Superlogger
     require 'superlogger/action_view_log_subscriber'
     require 'superlogger/active_record_log_subscriber'
   end
+
+  def session_id=(session_id)
+    RequestStore.store[:superlogger_session_id] = session_id
+  end
+
+  def session_id
+    RequestStore.store[:superlogger_session_id]
+  end
+
+  def request_id=(request_id)
+    RequestStore.store[:superlogger_request_id] = request_id
+  end
+
+  def request_id
+    RequestStore.store[:superlogger_request_id]
+  end
+
+  def enabled=(enabled)
+    @@enabled=enabled
+  end
+
+  def enabled
+    @@enabled
+  end
 end
 
 require 'superlogger/railtie' if defined?(Rails)

data/lib/superlogger/action_controller_log_subscriber.rb

@@ -5,28 +5,19 @@ module Superlogger
     # start of controller action
     def start_processing(event)
       payload = event.payload
-
-      Logger.debug controller: payload[:controller], action: payload[:action], params: payload[:params].except(*INTERNAL_PARAMS)
+      logger.debug {{ controller: payload[:controller], action: payload[:action], params: payload[:params].except(*INTERNAL_PARAMS) }}
     end
 
     # end of controller action
     def process_action(event)
       payload = event.payload
+      view_duration  = payload[:view_runtime].to_f.round(2) if payload.key?(:view_runtime)
+      db_duration    = payload[:db_runtime].to_f.round(2) if payload.key?(:db_runtime)
 
       if payload[:exception]
-        status = ActionDispatch::ExceptionWrapper.status_code_for_exception(payload[:exception][0])
-
-        Logger.fatal status: status, exception: payload[:exception]
+        logger.fatal view_duration: view_duration, db_duration: db_duration, exception: payload[:exception]
       else
-        # Assume status 401 if action finishes without status code and no exception
-        # https://github.com/pcg79/devise/commit/1e2dab3c0ce49efe2b5940c15f47388c69d6731b
-        payload[:status] ||= 401
-
-        total_duration = event.duration.to_f.round(2)
-        view_duration  = payload[:view_runtime].to_f.round(2) if payload.key?(:view_runtime)
-        db_duration    = payload[:db_runtime].to_f.round(2) if payload.key?(:db_runtime)
-
-        Logger.info status: payload[:status], total_duration: total_duration, view_duration: view_duration, db_duration: db_duration
+        logger.info view_duration: view_duration, db_duration: db_duration
       end
     end
   end

data/lib/superlogger/action_view_log_subscriber.rb

@@ -3,7 +3,7 @@ module Superlogger
     def render_template(event)
       payload = event.payload
 
-      Logger.debug view: payload[:identifier].split('/').last, duration: event.duration.round(1)
+      logger.debug {{ view: payload[:identifier].split('/').last, duration: event.duration.round(2) }}
     end
     alias :render_partial :render_template
     alias :render_collection :render_template

data/lib/superlogger/active_record_log_subscriber.rb

@@ -18,10 +18,12 @@ module Superlogger
       payload = event.payload
       return if IGNORE_PAYLOAD_NAMES.include?(payload[:name])
 
-      sql = payload[:sql]
-      params = payload[:binds].map { |_, value| "'#{value}'"}
+      logger.debug do
+        sql = payload[:sql]
+        params = payload[:binds].map { |b| b.value.to_s }
 
-      Logger.debug sql: sql, params: params, duration: event.duration.round(2)
+        { sql: sql, params: params, duration: event.duration.round(2) }
+      end
     end
   end
 end

data/lib/superlogger/logger.rb

@@ -1,79 +1,48 @@
 require 'request_store'
 
 module Superlogger
-  module Logger
-    def self.session_id=(session_id)
-      RequestStore.store[:superlogger_session_id] = session_id
-    end
+  class Logger < ActiveSupport::Logger
+    def format_message(severity, time, _progname, msg)
+      return nil if msg.blank? # Silence nil and empty msg
+      return nil if is_rails_rack_logger_msg?(msg) # Silence rack logger msg
 
-    def self.session_id
-      RequestStore.store[:superlogger_session_id] || "NS-#{Thread.current.object_id}"
-    end
+      msg = {msg: msg} unless msg.is_a?(Hash)
 
-    def self.debug(*args)
-      Rails.logger.debug { format_msg(args) }
-    end
-
-    def self.info(*args)
-      Rails.logger.info { format_msg(args) }
-    end
-
-    def self.warn(*args)
-      Rails.logger.warn { format_msg(args) }
-    end
-
-    def self.error(*args)
-      Rails.logger.error { format_msg(args) }
-    end
+      h = {}
+      h[:level] = severity.downcase
+      h[:ts] = time.to_f
+      h[:caller] = get_caller_location
+      h[:session_id] = Superlogger.session_id[0..11] unless Superlogger.session_id.nil?
+      h[:request_id] = Superlogger.request_id unless Superlogger.request_id.nil?
+      h.merge!(msg)
 
-    def self.fatal(*args)
-      Rails.logger.fatal { format_msg(args) }
+      h.to_json + "\n"
     end
 
-    private
-
-    def self.format_msg(args)
-      "#{get_caller_location} | #{format_args(args)}"
+    def is_rails_rack_logger_msg?(msg)
+      return false unless msg.is_a?(String)
+      msg =~ /Started (GET|POST|PUT|PATCH|DELETE)/
     end
 
-    def self.get_caller_location
-      # Find the last method call on Superlogger::Logger
-      count = 0
-      location_index = caller_locations(0).find_index do |location|
-        count += 1 if location && location.path.include?('superlogger/logger.rb')
-        count == 4
-      end
-
-      # Add 1 to get the caller of the logger
-      location = caller_locations(location_index + 1).first
+    def get_caller_location
+      index = caller_locations(4, 1).first.label.include?('broadcast') ? 6 : 5
+      location = caller_locations(index, 1).first
 
-      # extract filename without file extension from location.path
-      # eg. superlogger/lib/superlogger/logger.rb
-      file = location.path.split('/').last.split('.').first
+      # Extract filename without file extension from location.path
+      # eg. superlogger/lib/superlogger/logger.rb => superlogger/logger
+      file = location.path.split('/').last(2).join('/').split('.').first
 
       "#{file}:#{location.lineno}"
     end
 
-    def self.format_args(args)
-      # format args in key=value pair, separated by pipes
-      args.map do |arg|
-        arg = {nil: arg} unless arg.is_a?(Hash)
-
-        arg.map do |key, value|
-          "#{key}=#{value}"
+    # To silence double logging when running `rails server` in development mode
+    # See: https://github.com/rails/rails/commit/3d10d9d6c3b831fe9632c43a0ffec46104f912a7
+    if Rails.env.development?
+      class SimpleFormatter < ::Logger::Formatter
+        def call(_severity, _timestamp, _progname, _msg)
+          nil
         end
-      end.flatten.join(' | ')
+      end
     end
   end
 end
-
-# for overriding default Rails Logger format
-class Logger
-  def format_message(severity, time, _progname, msg)
-    timestamp = time.strftime('%Y-%m-%d %H:%M:%S.%L')
-    session_id = Superlogger::Logger.session_id[0..11]
-    severity = severity.to_s.upcase[0]
-
-    "#{timestamp} | #{session_id} | #{severity} | #{msg}\n"
-  end
-end

data/lib/superlogger/superlogger_middleware.rb

@@ -0,0 +1,47 @@
+module Superlogger
+  class SuperloggerMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      request = ActionDispatch::Request.new(env)
+      
+      if request.path.start_with?('/assets/') == false
+        process_request(request) { @app.call(env) }
+      else
+        @app.call(env)
+      end
+    end
+
+    def process_request(request)
+      setup_logging(request)
+
+      # Start of request
+      Rails.logger.info method: request.method, path: request.fullpath
+
+      t1 = Time.now
+      status, _headers, _response = yield
+    ensure
+      t2 = Time.now
+
+      # End of request
+      duration = ((t2 - t1) * 1000).to_f.round(2)
+      Rails.logger.info method: request.method, path: request.fullpath, response_time: duration, status: status
+
+      [status, _headers, _response]
+    end
+
+    def setup_logging(request)
+      if request.env['rack.session']
+        # Session is lazy loaded. Force session to load if it is not already loaded.
+        request.env['rack.session'].send(:load!) unless request.env['rack.session'].id
+
+        # Store session id before any actual logging is done
+        Superlogger.session_id = request.env['rack.session'].id.to_s
+      end
+
+      Superlogger.request_id = request.uuid.try(:gsub, '-', '')
+    end
+  end
+end