heavylog 0.0.19 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 817f124071c11b74c359b8ce78ec24cc8778bfab7c3d3ee79da64847e30f1f66
-  data.tar.gz: 80ecb337b3e34dc5a50419f31d3a9bb20fdddbe7da23ffefc52b9b2197eee757
+  metadata.gz: 0d938340a9fc57747dc46c35aadb0a9b92c08da6466f3d333479e7b0c4de63d4
+  data.tar.gz: f31b3c1ec5566d8de8c6199b21dfae45b00b22c5710dd1eecb07e676a55b3a2e
 SHA512:
-  metadata.gz: 54e8d27f4dd9c4e69bd4e2a8103a085e246f0e9067bdd056c328efd5edf73d6c9158626bd68245bc7ace3bedfe58ef6a8c821de034d3737d0670231f2cbf47e1
-  data.tar.gz: ade91b07e69e29c288b5cda82b3dd00717248099bbf7da75e9f375e08732d53a506f9345f2565ea349853eff66a1c1de65219160c70949fba532e6a49323d463
+  metadata.gz: 44fa9815cb453c895d70db9912558f9b16cf775caaa83d1d723fde9252ced6a6c4a07072d0b0db8b352236a11701330af9ec0adf7de8bc94c41a89e96c6e5dfa
+  data.tar.gz: 993f3cb991245997f89827c4bd6581060b35eadd64d4c229419c5dbe6cbc1f8467319e66514c91af9d445bd683231144d8393419729b4ab513f7a7996ffe1854

data/.github/workflows/test.yml

@@ -1,6 +1,6 @@
 name: Test
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   test:
@@ -10,9 +10,9 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Set up Ruby
-        uses: actions/setup-ruby@v1
+        uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0.0
+          ruby-version: 3.0.1
 
       - uses: actions/cache@v1
         with:
@@ -28,9 +28,6 @@ jobs:
           bundle install --jobs 4 --retry 3
 
       - name: Run tests
-        uses: paambaati/codeclimate-action@v2.4.0
         env:
           CI: true
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        with:
-          coverageCommand: bundle exec rspec
+        run: bundle exec rspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    heavylog (0.0.19)
+    heavylog (0.0.20)
       actionpack (>= 5)
       activesupport (>= 5)
       railties (>= 5)
@@ -10,20 +10,20 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.1.3.1)
-      actionview (= 6.1.3.1)
-      activesupport (= 6.1.3.1)
+    actionpack (6.1.3.2)
+      actionview (= 6.1.3.2)
+      activesupport (= 6.1.3.2)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.1.3.1)
-      activesupport (= 6.1.3.1)
+    actionview (6.1.3.2)
+      activesupport (= 6.1.3.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activesupport (6.1.3.1)
+    activesupport (6.1.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -34,7 +34,7 @@ GEM
     benchmark (0.1.1)
     builder (3.2.4)
     concurrent-ruby (1.1.8)
-    connection_pool (2.2.3)
+    connection_pool (2.2.5)
     crass (1.0.6)
     diff-lcs (1.4.4)
     docile (1.3.5)
@@ -52,13 +52,13 @@ GEM
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.14.4)
-    nokogiri (1.11.2)
+    nokogiri (1.11.3)
       mini_portile2 (~> 2.5.0)
       racc (~> 1.4)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.1.0)
       ast (~> 2.4.1)
     racc (1.5.2)
     rack (2.2.3)
@@ -69,9 +69,9 @@ GEM
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.1.3.1)
-      actionpack (= 6.1.3.1)
-      activesupport (= 6.1.3.1)
+    railties (6.1.3.2)
+      actionpack (= 6.1.3.2)
+      activesupport (= 6.1.3.2)
       method_source
       rake (>= 0.8.7)
       thor (~> 1.0)
@@ -83,7 +83,7 @@ GEM
       rack (>= 1.4)
     reverse_markdown (2.0.0)
       nokogiri
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -111,7 +111,7 @@ GEM
     rubocop-performance (1.5.2)
       rubocop (>= 0.71.0)
     ruby-progressbar (1.11.0)
-    sidekiq (6.2.0)
+    sidekiq (6.2.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)

data/README.md

@@ -1,39 +1,102 @@
-# Heavylog [![Build Status](https://travis-ci.org/krisrang/heavylog.svg?branch=master)](https://travis-ci.org/krisrang/heavylog)
+# Heavylog [![Test](https://github.com/krisrang/heavylog/actions/workflows/test.yml/badge.svg)](https://github.com/krisrang/heavylog/actions/workflows/test.yml) [![Gem Version](https://badge.fury.io/rb/heavylog.svg)](https://badge.fury.io/rb/heavylog)
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/heavylog`. To experiment with that code, run `bin/console` for an interactive prompt.
+Heavylog enables you to log all Rails requests to file as JSON or any other format you want.
 
-TODO: Delete this and the text above, and describe your gem
+Aside from metadata about requests full output is also included like SQL/Rails logging and `puts` statements.  
+Example request using the JSON formatter:
+```
+{"request_id":"e2cdef0a-9851-4aab-b58f-60e607b4d1a9","request_start":"2021-04-25T15:37:20+00:00","ip":"52.52.52.52","messages":"Started GET \"/admin/info/sidekiq\" for 52.52.52.52 at 2021-04-25 15:37:20 +0000\nProcessing by Admin::InfoController#sidekiq_stats as */*\nRequested via apphost.com as */*\n  Snippet Load (1.8ms)  SELECT \"snippets\".* FROM \"snippets\" WHERE (locale = 'sv-SE' AND tag = 'information_notice_contact') ORDER BY \"snippets\".\"id\" ASC LIMIT $1  [[\"LIMIT\", 1]]\n  Snippet Load (1.5ms)  SELECT \"snippets\".* FROM \"snippets\" WHERE (locale = 'sv-SE' AND tag = 'contact_us_frame') ORDER BY \"snippets\".\"id\" ASC LIMIT $1  [[\"LIMIT\", 1]]\n  Rendering text template\n  Rendered text template (Duration: 0.1ms | Allocations: 16)\nCompleted 200 OK in 41ms (Views: 0.6ms | ActiveRecord: 3.3ms | Allocations: 10734)\n","method":"GET","path":"/admin/info/sidekiq","format":"*/*","controller":"Admin::InfoController","action":"sidekiq_stats","status":200,"duration":40.74,"view_runtime":0.56,"db_runtime":3.28,"user_id":null,"admin_id":null,"request_host":"apphost.com","ua":"curl/7.58.0","operation":null}
+```
 
+Example use case is collecting the JSON files and shipping them to an Elastic/Kibana cluster for centralized logging.
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'heavylog'
+gem "heavylog"
 ```
 
 And then execute:
 
-    $ bundle
+```bash
+$ bundle
+```
 
-Or install it yourself as:
+## Usage
 
-    $ gem install heavylog
+Enable and configure in a Rails initializer `config/initializers/logger.rb`:
 
-## Usage
+```rb
+Rails.application.configure do
+  config.heavylog.enabled = true
+  config.heavylog.path = Rails.root.join("log/heavylog.log")
+  # Default formatter is Heavylog::Formatters::Raw which simply outputs the ruby hash as a string
+  config.heavylog.formatter = Heavylog::Formatters::Json.new
+  config.heavylog.log_sidekiq = true # Default is `false`, set to `true` to automatically log sidekiq job runs too.
+end
+```
 
-TODO: Write usage instructions here
+## Configuration
+
+| Option | Type     | Description                |
+| :-------- | :------- | :------------------------- |
+| `enabled` | `boolean` | Set to `true` to enable logging. Default: `false`.  |
+| `path` | `string` | Path to output file. Default: `log/heavylog.log`.  |
+| `message_limit` | `integer` | Request output will be truncated if longer than this limit. Default: `52428800` (50MB).  |
+| `log_sidekiq` | `boolean` | Set to `true` to automatically log sidekiq jobs too. Default: `false`.  |
+| `error_handler` | `lambda/proc` | Code to execute when formatter throws exception. Default: `->(e) { p "HeavyLog: Error writing to log: #{e.class}: #{e.message}\n  #{e.backtrace.join("\n  ")}" }`
+| `custom_payload` | `block` | Block executed for every request that should return hash with extra fields you want to log. Default: `nil`. |
+
+### Custom payload
+
+`custom_payload` accepts a block with a single argument, `controller` you can use to access any methods you normally would in a controller action.  
+It should return a hash with the extra fields you want to log.
+
+```rb
+Rails.application.configure do
+  config.heavylog.custom_payload do |controller|
+    user_id = controller.respond_to?(:current_user) ? controller.current_user&.id : nil
+
+    {
+      user_id:      user_id,
+      request_host: controller.request.host,
+      ua:           controller.request.user_agent,
+      operation:    controller.request.params[:operationName],
+    }
+  end
+end
+```
 
-## Development
+### Sidekiq logging
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Set `log_sidekiq` to `true` if you want to automatically log Sidekiq job runs to the same file. Example with JSON formatter:
+```
+{"request_id":"fb2c3798e2634011d670f753","request_start":"2021-04-25T16:00:53+00:00","ip":"127.0.0.1","messages":"  Order Load (1.8ms)  SELECT \"orders\".* FROM \"orders\" WHERE \"orders\".\"id\" = $1 LIMIT $2  [[\"id\", 109987473], [\"LIMIT\", 1]]\n  Customer Load (1.7ms)  SELECT \"customers\".* FROM \"customers\" WHERE \"customers\".\"id\" = $1 LIMIT $2  [[\"id\", 1027337], [\"LIMIT\", 1]]\n","controller":"SidekiqLogger","action":"MailPrepaidCheckupsJob","args":"[109987473]"}
+```
+
+Sidekiq job runs go into the same file as regular request logs but have controller set to `SidekiqLogger` and action to the name of the Job. 
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+### JSON formatter
 
-## Contributing
+Every request results in a hash containing all info about the request. The default `Heavylog::Formatters::Raw` formatter simply outputs the hash as a string to the output file.  
+Use the `Heavylog::Formatters::Json` formatter to convert the hash to JSON. The resulting file will contain one JSON object per line for every request.
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/krisrang/heavylog.
+### Custom formatter
 
+The formatter interface is simply a class with a `call` method that accepts a single argument which is the hash containing info about the request.  
+The method should return the final result you want to write to file. Heavylog writes one line per request.
+
+JSON formatter for example:
+```rb
+class Json
+  def call(data)
+    ::JSON.dump(data)
+  end
+end
+```
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+[MIT](https://choosealicense.com/licenses/mit/)
+
+  

data/lib/heavylog.rb

@@ -3,6 +3,7 @@
 require "heavylog/version"
 require "heavylog/formatters/raw"
 require "heavylog/formatters/json"
+require "heavylog/formatters/ecs"
 require "heavylog/log_subscriber"
 require "heavylog/middleware"
 require "heavylog/ordered_options"
@@ -136,7 +137,7 @@ module Heavylog
       messages:      buffer.string.dup,
     }.merge(RequestStore.store[:heavylog_request_data] || {})
 
-    formatted = Heavylog.formatter.call(request)
+    formatted = Heavylog.formatter.call(request.transform_keys(&:to_s))
     Heavylog.logger.send(Heavylog.log_level, formatted)
   rescue StandardError => e
     config.error_handler&.(e)

data/lib/heavylog/formatters/ecs.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Heavylog
+  module Formatters
+    class ECS
+      # mapping from heavylog standard keys to ECS https://www.elastic.co/guide/en/ecs/current/ecs-reference.html
+      ECS_MAP = {
+        "request_start"      => "@timestamp",
+        "messages"           => "message",
+        "request_id"         => "http.request.id",
+        "method"             => "http.request.method",
+        "format"             => "http.request.mime_type",
+        "status"             => "http.response.status_code",
+        "location"           => "http.response.location",
+        "ip"                 => "source.address",
+        "path"               => "url.original",
+        "controller"         => "heavylog.controller",
+        "action"             => "heavylog.action",
+        "unpermitted_params" => "heavylog.unpermitted_params",
+        "args"               => "heavylog.args",
+        "duration"           => "heavylog.duration",
+        "view_runtime"       => "heavylog.view_runtime",
+        "db_runtime"         => "heavylog.db_runtime",
+      }.freeze
+
+      def call(data)
+        ECS_MAP.each do |original, correct|
+          data[correct] = data.delete(original) if data[original]
+        end
+
+        data["event.module"] = "heavylog"
+        data["event.category"] = "web"
+        data["event.dataset"] ||= data["heavylog.controller"] == "SidekiqLogger" ? "heavylog.sidekiq" : "heavylog.rails"
+
+        if data["http.response.status_code"]
+          data["event.outcome"] = (200..399).cover?(data["http.response.status_code"]) ? "success" : "failure"
+        end
+
+        data["source.ip"] ||= data["source.address"]
+        data["url.path"] ||= data["url.original"]
+
+        ::JSON.dump(data)
+      end
+    end
+  end
+end

data/lib/heavylog/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Heavylog
-  VERSION = "0.0.19"
+  VERSION = "0.0.20"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heavylog
 version: !ruby/object:Gem::Version
-  version: 0.0.19
+  version: 0.0.20
 platform: ruby
 authors:
 - Kristjan Rang
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-25 00:00:00.000000000 Z
+date: 2021-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -199,6 +199,7 @@ files:
 - bin/setup
 - heavylog.gemspec
 - lib/heavylog.rb
+- lib/heavylog/formatters/ecs.rb
 - lib/heavylog/formatters/json.rb
 - lib/heavylog/formatters/raw.rb
 - lib/heavylog/log_subscriber.rb