superlogger 0.2.1 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 95512203bc8b39f0d4924b68d77e97ffc79c26e6
-  data.tar.gz: 19ae2427a6b9e1df1b85ba7ed252fb0ae98d424e
+SHA256:
+  metadata.gz: '082cbc716a6f0432e73617a883fd03584bd8f3de5a1f3f32afd6a48882d05f76'
+  data.tar.gz: c85c50a9e182a78c3c281a427b5fd351c4c01b9181dbeffe81a6831bc51f7b39
 SHA512:
-  metadata.gz: 73305d6f09e850f0a1534f8836db120bd20bf46a843330b38226d3778cb3044d73d31bc1e5473b5c7308465b3d75062ca8705bc2ce23ded18f2bcd17c2652b3f
-  data.tar.gz: 25807a67f277bbdfcddf590c705bad9a3ef3eb16cad72c45019db16626fc994fae1ec1e676fda5a0be2369dc637f8cf22798539e11d6c8fe46b0a0127f88133b
+  metadata.gz: 7f5eb670792f8cb1abf4a4b3830b7b0e415b9f0849d9339bf8ddce091dc71dd7f2cc455813f7c6ca64fafba40ab8de59f32c6fd1a6c5cec74d24968caecc3624
+  data.tar.gz: bc3c65ddb32e7e5eec96de1028d86a12557fda30643797eb71e863db4dac24fcf82dd5434a8c72639a9eb78b1168e60fc4c2dc3fbbd71fffb6a0dcbf08cc3f13

data/README.md

@@ -5,7 +5,7 @@
 Superlogger - Machine-readable logging for Rails
 =======
 
-Rails' default request logging is easy to read for humans but difficult for log aggregators such as Kibana, Graylog and Splunk. Superlogger transforms the logs into key-value pairs for easy parsing and adds useful details like Timestamp, Session ID and Request ID for tracing purposes.
+Rails' default request logging is easy to read for humans but difficult for log aggregators such as Kibana, Graylog and Splunk. Superlogger transforms the logs into JSON for easy parsing and adds useful details like Timestamp, Session ID and Request ID for tracing purposes.
 
 Default rails logging:
 ```sh
@@ -23,34 +23,18 @@ Started GET "/assets/application.self-8f06a73c35179188914ab50e057157639fce1401c1
 
 ```
 
-Machine-readable logging with Superlogger:
+With Superlogger:
 ```sh
-# first request
-2016-04-29 17:29:45.841 | 12dc0e484869 | 68d63a8cf920 | I | superlogger_middleware:30 | method=GET | path=/home/index | ip=::1
-2016-04-29 17:29:45.847 | 12dc0e484869 | 68d63a8cf920 | D | action_controller_log_subscriber:9 | controller=HomeController | action=index | params={}
-2016-04-29 17:29:45.852 | 12dc0e484869 | 68d63a8cf920 | D | active_record_log_subscriber:24 | sql=SELECT  "somethings".* FROM "somethings" WHERE "somethings"."paper" = ? AND "somethings"."stone" = ?  ORDER BY "somethings"."id" ASC LIMIT 1 | params=["123", "456"] | duration=0.13
-2016-04-29 17:29:45.861 | 12dc0e484869 | 68d63a8cf920 | D | action_view_log_subscriber:6 | view=_partial.html.erb | duration=0.2
-2016-04-29 17:29:45.861 | 12dc0e484869 | 68d63a8cf920 | D | action_view_log_subscriber:6 | view=index.html.erb | duration=3.2
-2016-04-29 17:29:45.983 | 12dc0e484869 | 68d63a8cf920 | I | action_controller_log_subscriber:29 | status=200 | total_duration=135.92 | view_duration=130.38 | db_duration=0.33
-2016-04-29 17:29:45.983 | 12dc0e484869 | 68d63a8cf920 | I | superlogger_middleware:30 | method=GET | path=/home/index | total_duration=135.92
-
-# second request
-2016-04-29 17:39:54.879 | 12dc0e484869 | e463d380fb63 | I | superlogger_middleware:30 | method=GET | path=/home/show | ip=::1
-2016-04-29 17:39:54.879 | 12dc0e484869 | e463d380fb63 | D | action_controller_log_subscriber:9 | controller=HomeController | action=show | params={}
-2016-04-29 17:39:54.882 | 12dc0e484869 | e463d380fb63 | D | action_view_log_subscriber:6 | view=show.html.erb | duration=0.2
-2016-04-29 17:39:54.884 | 12dc0e484869 | e463d380fb63 | I | action_controller_log_subscriber:29 | status=200 | total_duration=4.64 | view_duration=4.55 | db_duration=0.0
-2016-04-29 17:39:54.884 | 12dc0e484869 | e463d380fb63 | I | superlogger_middleware:30 | method=GET | path=/home/index | total_duration=4.64
+{"level":"debug","ts":1590972589.522784,"caller":"superlogger/active_record_log_subscriber:21","sql":"SELECT \"schema_migrations\".\"version\" FROM \"schema_migrations\" ORDER BY \"schema_migrations\".\"version\" ASC","params":[],"duration":0.13}
+{"level":"info","ts":1590972589.526133,"caller":"superlogger/superlogger_middleware:21","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","method":"GET","path":"/home/index"}
+{"level":"debug","ts":1590972589.546272,"caller":"superlogger/action_controller_log_subscriber:8","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","controller":"HomeController","action":"index","params":{}}
+{"level":"debug","ts":1590972589.55092,"caller":"superlogger/active_record_log_subscriber:21","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","sql":"SELECT  \"somethings\".* FROM \"somethings\" WHERE \"somethings\".\"paper\" = ? AND \"somethings\".\"stone\" = ? ORDER BY \"somethings\".\"id\" ASC LIMIT ?","params":["123","456","1"],"duration":0.22}
+{"level":"debug","ts":1590972589.574199,"caller":"superlogger/action_view_log_subscriber:6","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view":"_partial.html.erb","duration":0.33}
+{"level":"debug","ts":1590972589.574795,"caller":"superlogger/action_view_log_subscriber:6","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view":"index.html.erb","duration":2.95}
+{"level":"info","ts":1590972589.61165,"caller":"superlogger/action_controller_log_subscriber:20","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","view_duration":54.59,"db_duration":0.85}
+{"level":"info","ts":1590972589.611928,"caller":"superlogger/superlogger_middleware:30","session_id":"90e90c75c72c","request_id":"34432478c89b4d8591e02e0169b40a56","method":"GET","path":"/home/index","response_time":85.65,"status":200}
 ```
 
-## Features ##
-- Timestamp (milliseconds)
-- Session ID for logs belonging to the same user session (notice above that both the requests have the same session id)
-- Request ID for logs belonging to the same page request (notice above that each request have a different request id)
-- Hashes will be logged as key-value pairs automatically
-- Requests for assets will not be logged
-- File and line numbers 
-- IP address of request
-
 ## Installation ##
 
 Add superlogger to your application's Gemfile
@@ -63,37 +47,28 @@ Execute:
 $ bundle
 ```
 
-And add the following in `config/application.rb`
+And add the following in `config/environment/production.rb`
 ```ruby
 config.logger = Superlogger::Logger.new(STDOUT)
 ```
 
+By default, Superlogger is only enabled in production environment because JSON is easy for machines to parse but difficult for humans to read. To forcefully enable Superlogger in non-production environment, set in `config/application.rb`:
+```ruby
+Superlogger.enabled = true
+```
+
 ## Usage ##
 
 Log as per normal using `Rails.logger`.
 
 ```ruby
-class SomeClass
-    def some_method
-        Rails.logger.debug foo:'true', bar: 'false'
-        Rails.logger.info foo:'true', bar: 'false'
-        Rails.logger.warn foo:'true', bar: 'false'
-        Rails.logger.error foo:'true', bar: 'false'
-        Rails.logger.fatal foo:'true', bar: 'false'
-    end
-end
+Rails.logger.info foo:'true', bar: 'false'
+Rails.logger.info "Meatball"
 ```
 
 ## Log Format ##
-```sh
-2015-03-26 23:37:38.086 | 12dc0e484869   | e463d380fb63   | I            | action_controller_log_subscriber:29 | status=200 | total_duration=4.64 | view_duration=4.55 | db_duration=0.0
-< timestamp >           | < session id > | < request id > | < severity > | < file >:< line num >               | < data you pass in ... >
-< 23 chars >            | < 12 chars >   | <12 chars >    | < 1 char >   | < ? chars >                         | < ? chars >
-```
-
-### Severity Levels ###
-- **D** - Debug
-- **I** - Info
-- **W** - Warn
-- **E** - Error
-- **F** - Fatal
+- `ts` = Unix Epoch timestamp
+- `session_id` = Truncated to 12 characters
+- `request_id` = 32 characters
+- `msg` = If values given is not a hash, it is treated as `{"msg":<value>"}`
+- All duration related fields are in milliseconds

data/lib/superlogger/action_controller_log_subscriber.rb

@@ -5,27 +5,19 @@ module Superlogger
     # start of controller action
     def start_processing(event)
       payload = event.payload
-
-      logger.debug controller: payload[:controller], action: payload[:action], params: payload[:params].except(*INTERNAL_PARAMS)
+      logger.debug {{ controller: payload[:controller], action: payload[:action], params: payload[:params].except(*INTERNAL_PARAMS) }}
     end
 
     # end of controller action
     def process_action(event)
       payload = event.payload
-      total_duration = event.duration.to_f.round(2)
       view_duration  = payload[:view_runtime].to_f.round(2) if payload.key?(:view_runtime)
       db_duration    = payload[:db_runtime].to_f.round(2) if payload.key?(:db_runtime)
 
       if payload[:exception]
-        status = ActionDispatch::ExceptionWrapper.status_code_for_exception(payload[:exception][0])
-
-        logger.fatal status: status, total_duration: total_duration, view_duration: view_duration, db_duration: db_duration, exception: payload[:exception]
+        logger.fatal view_duration: view_duration, db_duration: db_duration, exception: payload[:exception]
       else
-        # Assume status 401 if action finishes without status code and no exception
-        # https://github.com/pcg79/devise/commit/1e2dab3c0ce49efe2b5940c15f47388c69d6731b
-        payload[:status] ||= 401
-
-        logger.info status: payload[:status], total_duration: total_duration, view_duration: view_duration, db_duration: db_duration
+        logger.info view_duration: view_duration, db_duration: db_duration
       end
     end
   end

data/lib/superlogger/action_view_log_subscriber.rb

@@ -3,7 +3,7 @@ module Superlogger
     def render_template(event)
       payload = event.payload
 
-      logger.debug view: payload[:identifier].split('/').last, duration: event.duration.round(1)
+      logger.debug {{ view: payload[:identifier].split('/').last, duration: event.duration.round(2) }}
     end
     alias :render_partial :render_template
     alias :render_collection :render_template

data/lib/superlogger/active_record_log_subscriber.rb

@@ -18,10 +18,12 @@ module Superlogger
       payload = event.payload
       return if IGNORE_PAYLOAD_NAMES.include?(payload[:name])
 
-      sql = payload[:sql]
-      params = payload[:binds].map { |_, value| value.to_s }
+      logger.debug do
+        sql = payload[:sql]
+        params = payload[:binds].map { |b| b.value.to_s }
 
-      logger.debug sql: sql, params: params, duration: event.duration.round(2)
+        { sql: sql, params: params, duration: event.duration.round(2) }
+      end
     end
   end
 end

data/lib/superlogger/logger.rb

@@ -2,26 +2,25 @@ require 'request_store'
 
 module Superlogger
   class Logger < ActiveSupport::Logger
-    def initialize(*args)
-      super
-      @formatter = SimpleFormatter.new
-    end
-
     def format_message(severity, time, _progname, msg)
       return nil if msg.blank? # Silence nil and empty msg
       return nil if is_rails_rack_logger_msg?(msg) # Silence rack logger msg
 
-      timestamp = time.strftime('%Y-%m-%d %H:%M:%S.%L')
-      session_id = Superlogger.session_id[0..11]
-      request_id = Superlogger.request_id[0..11]
-      severity = severity.to_s.upcase[0]
-      caller_location = get_caller_location
-      args = format_args(msg)
+      msg = {msg: msg} unless msg.is_a?(Hash)
+
+      h = {}
+      h[:level] = severity.downcase
+      h[:ts] = time.to_f
+      h[:caller] = get_caller_location
+      h[:session_id] = Superlogger.session_id[0..11] unless Superlogger.session_id.nil?
+      h[:request_id] = Superlogger.request_id unless Superlogger.request_id.nil?
+      h.merge!(msg)
 
-      "#{timestamp} | #{session_id} | #{request_id} | #{severity} | #{caller_location} | #{args}\n"
+      h.to_json + "\n"
     end
 
     def is_rails_rack_logger_msg?(msg)
+      return false unless msg.is_a?(String)
       msg =~ /Started (GET|POST|PUT|PATCH|DELETE)/
     end
 
@@ -30,25 +29,12 @@ module Superlogger
       location = caller_locations(index, 1).first
 
       # Extract filename without file extension from location.path
-      # eg. superlogger/lib/superlogger/logger.rb
-      file = location.path.split('/').last.split('.').first
+      # eg. superlogger/lib/superlogger/logger.rb => superlogger/logger
+      file = location.path.split('/').last(2).join('/').split('.').first
 
       "#{file}:#{location.lineno}"
     end
 
-    def format_args(args)
-      output = if args.is_a?(Hash)
-                 # Format args in key=value pair, separated by pipes
-                 args.map do |key, value|
-                   "#{key}=#{value}"
-                 end.join(' | ')
-               else
-                 args.to_s
-               end
-
-      output.gsub("\n", '\\n') # Escape newlines
-    end
-
     # To silence double logging when running `rails server` in development mode
     # See: https://github.com/rails/rails/commit/3d10d9d6c3b831fe9632c43a0ffec46104f912a7
     if Rails.env.development?

data/lib/superlogger/superlogger_middleware.rb

@@ -6,7 +6,7 @@ module Superlogger
 
     def call(env)
       request = ActionDispatch::Request.new(env)
-
+      
       if request.path.start_with?('/assets/') == false
         process_request(request) { @app.call(env) }
       else
@@ -18,27 +18,26 @@ module Superlogger
       setup_logging(request)
 
       # Start of request
-      Rails.logger.info method: request.method, path: request.fullpath, ip: request.ip
+      Rails.logger.info method: request.method, path: request.fullpath
 
       t1 = Time.now
-      result = yield
+      status, _headers, _response = yield
     ensure
       t2 = Time.now
 
       # End of request
       duration = ((t2 - t1) * 1000).to_f.round(2)
-      Rails.logger.info method: request.method, path: request.fullpath, total_duration: duration
+      Rails.logger.info method: request.method, path: request.fullpath, response_time: duration, status: status
 
-      result
+      [status, _headers, _response]
     end
 
     def setup_logging(request)
       if request.env['rack.session']
-        # Session is lazy loaded. Force session to load if it is not already loaded.
-        request.env['rack.session'].send(:load!) unless request.env['rack.session'].id
-
         # Store session id before any actual logging is done
-        Superlogger.session_id = request.env['rack.session'].id
+        if request.env['rack.session'].id
+          Superlogger.session_id = request.env['rack.session'].id.to_s
+        end
       end
 
       Superlogger.request_id = request.uuid.try(:gsub, '-', '')

data/lib/superlogger/version.rb

@@ -1,3 +1,3 @@
 module Superlogger
-  VERSION = '0.2.1'
+  VERSION = '1.0.1'
 end

data/lib/superlogger.rb

@@ -2,9 +2,13 @@ require 'superlogger/version'
 require 'superlogger/logger'
 
 module Superlogger
+  @@enabled = false
+
   module_function
 
   def setup(app)
+    return unless (Rails.env.production? || enabled)
+    
     insert_superlogger_middleware(app)
     detach_existing_log_subscribers
     attach_superlogger_log_subscribers
@@ -24,7 +28,7 @@ module Superlogger
     require 'action_view/log_subscriber'
 
     # remove log subscribers
-    patterns = %w(sql.active_record
+    remove_patterns = %w(sql.active_record
                   start_processing.action_controller
                   process_action.action_controller
                   render_template.action_view
@@ -32,8 +36,11 @@ module Superlogger
                   render_collection.action_view)
 
     ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
-      subscriber.patterns.each do |pattern|
-        ActiveSupport::Notifications.unsubscribe pattern if patterns.include?(pattern)
+      patterns = subscriber.patterns
+      patterns = patterns.is_a?(Hash) ? patterns.keys : patterns
+
+      patterns.each do |pattern|
+        ActiveSupport::Notifications.unsubscribe pattern if remove_patterns.include?(pattern)
       end
     end
   end
@@ -49,7 +56,7 @@ module Superlogger
   end
 
   def session_id
-    RequestStore.store[:superlogger_session_id] || "NS-#{Thread.current.object_id}"
+    RequestStore.store[:superlogger_session_id]
   end
 
   def request_id=(request_id)
@@ -57,7 +64,15 @@ module Superlogger
   end
 
   def request_id
-    RequestStore.store[:superlogger_request_id] || "NR-#{Thread.current.object_id}"
+    RequestStore.store[:superlogger_request_id]
+  end
+
+  def enabled=(enabled)
+    @@enabled=enabled
+  end
+
+  def enabled
+    @@enabled
   end
 end
 

data/test/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../javascripts .js
+//= link_directory ../stylesheets .css

data/test/dummy/config/application.rb

@@ -19,8 +19,9 @@ module Dummy
     # config.i18n.default_locale = :de
 
     # Do not swallow errors in after_commit/after_rollback callbacks.
-    config.active_record.raise_in_transactional_callbacks = true
-
+    # config.active_record.raise_in_transactional_callbacks = true
+    
+    Superlogger.enabled = true
     config.logger = Superlogger::Logger.new(STDOUT)
   end
 end

data/test/superlogger_test.rb

@@ -25,7 +25,7 @@ class SuperloggerTest < ActiveSupport::TestCase
 
   def output
     @parsed_output ||= @output.string.split("\n").map do |line|
-      line.split(' | ')
+      JSON.parse(line)
     end
   end
 
@@ -37,128 +37,100 @@ class SuperloggerTest < ActiveSupport::TestCase
     request('home/index')
 
     fields = output.first
-    assert fields[0].length == 23
-    assert fields[1].length == 12
-    assert fields[2].length == 12
-    assert fields[3].length == 1
-    assert_match(/\w:\d+/, fields[4])
+    assert fields.key?("level")
+    assert fields.key?("ts")
+    assert fields.key?("caller")
+    assert fields.key?("session_id")
+    assert fields.key?("request_id")
   end
 
   test 'timestamp' do
     request('home/index')
-    assert DateTime.parse(output[0][0]).to_date == Date.today
+    assert_equal Time.at(output[0]["ts"]).to_date, Date.today
   end
 
   test 'with session_id' do
     env = request('home/index')
-    assert_match env['rack.session'].id[0..11], output[0][1]
+    assert_match env['rack.session'].id.to_s[0..11], output[0]["session_id"]
   end
 
   test 'without session_id' do
     Rails.logger.debug var: 'test'
-    assert_match(/^NS-[[:alnum:]]+$/, output[0][1])
+    assert_equal output[0].key?("session_id"), false
   end
 
   test 'with request_id' do
     request('home/index')
-    assert_no_match(/^NR-[[:alnum:]]+$/, output[0][2])
+    assert_no_match(/^NR-[[:alnum:]]+$/, output[0]["request_id"])
   end
 
   test 'without request_id' do
     Rails.logger.debug var: 'test'
-    assert_match(/^NR-[[:alnum:]]+$/, output[0][2])
-  end
-
-  test 'log levels' do
-    Rails.logger.debug var: 'test'
-    Rails.logger.info var: 'test'
-    Rails.logger.warn var: 'test'
-    Rails.logger.error var: 'test'
-    Rails.logger.fatal var: 'test'
-
-    assert output[0][3] == 'D'
-    assert output[1][3] == 'I'
-    assert output[2][3] == 'W'
-    assert output[3][3] == 'E'
-    assert output[4][3] == 'F'
+    assert_equal output[0].key?("request_id"), false
   end
 
   test 'caller location' do
     Rails.logger.debug var: 'test'
-    assert output[0][4] == "superlogger_test:#{__LINE__ - 1}"
+    assert_equal output[0]["caller"], "test/superlogger_test:#{__LINE__ - 1}"
   end
 
   test 'silence_rails_rack_logger' do
     request('home/index')
-    assert_no_match 'Started GET', output[0][5]
+    assert_no_match 'Started GET', output[0]["msg"]
   end
 
   test 'middleware start' do
     request('home/index', 'REMOTE_ADDR' => '::1')
 
     fields = output[0]
-    assert_match 'method=GET', fields[5]
-    assert_match 'path=/home/index', fields[6]
-    assert_match 'ip=::1', fields[7]
+    assert_match 'GET', fields["method"]
+    assert_match '/home/index', fields["path"]
   end
 
   test 'middleware end' do
     request('home/index', 'REMOTE_ADDR' => '::1')
 
     fields = output.last
-    assert_match 'method=GET', fields[5]
-    assert_match 'path=/home/index', fields[6]
-    assert_match(/^total_duration=\d+.\d+$/, fields[7])
-    assert_operator fields[7].split('=').last.to_f, :>, 0
+    assert_match 'GET', fields["method"]
+    assert_match '/home/index', fields["path"]
+    assert_operator fields["response_time"], :>, 0
   end
 
   test 'action_controller_log_subscriber.start_processing' do
     request('home/index?a=1')
 
     fields = output[1]
-    assert_match 'controller=HomeController', fields[5]
-    assert_match 'action=index', fields[6]
-    assert_match 'params={"a"=>"1"}', fields[7]
+    assert_match 'HomeController', fields["controller"]
+    assert_match 'index', fields["action"]
+    assert_equal Hash.new.tap {|h| h["a"] ="1"}, fields["params"]
   end
 
   test 'action_controller_log_subscriber.process_action' do
     request('home/index')
 
     fields = output[5]
-    assert_match 'status=200', fields[5]
-    assert_match(/^total_duration=\d+.\d+$/, fields[6])
-    assert_operator fields[6].split('=').last.to_f, :>, 0
-    assert_match(/^view_duration=\d+.\d+$/, fields[7])
-    assert_operator fields[7].split('=').last.to_f, :>, 0
-    assert_match(/^db_duration=\d+.\d+$/, fields[8])
-    assert_operator fields[8].split('=').last.to_f, :>, 0
+    assert_operator fields["view_duration"], :>, 0
+    assert_operator fields["db_duration"], :>, 0
   end
 
   test 'action_view_log_subscriber.render_template.render_partial.render_collection' do
     request('home/index')
 
     fields = output[3]
-    assert_match 'view=_partial.html.erb', fields[5]
-    assert_match 'duration=', fields[6]
+    assert_match 'partial.html.erb', fields["view"]
+    assert fields.key?("duration")
 
     fields = output[4]
-    assert_match 'view=index.html.erb', fields[5]
-    assert_match 'duration=', fields[6]
+    assert_match 'index.html.erb', fields["view"]
+    assert fields.key?("duration")
   end
 
   test 'active_record_log_subscriber.sql' do
     request('home/index')
 
     fields = output[2]
-    assert_match 'sql=SELECT', fields[5]
-    assert_match 'params=[', fields[6]
-    assert_match(/^duration=\d+.\d+$/, fields[7])
-    assert_operator fields[7].split('=').last.to_f, :>, 0
-  end
-
-  test 'escape new lines' do
-    Rails.logger.debug var: 'first\nsecond'
-
-    assert_match 'first\\nsecond', output[0].last
+    assert_match 'SELECT', fields["sql"]
+    assert_equal ["123", "456", "1"], fields["params"]
+    assert_operator fields["duration"], :>, 0
   end
 end