httplog 1.3.0 → 1.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd063a60dafd4b6bcc29fa80d877f8bd650375f4633fb0f0bc4398075677b583
-  data.tar.gz: 136a0fe30f033ad30f8867e327e18109255735b371cc9ab5a7d8e973d3b6e201
+  metadata.gz: 9ce910d7c04847c28f836de26906c0c50aa4642b553b298d15067bf6ee25db56
+  data.tar.gz: 8600e756f28e033b6e39ac91235f69bf772fe467b05040d88c02ce54e982ef0b
 SHA512:
-  metadata.gz: d4de29f881f37f1149400324e8c97de77dd7d7ef21e983dd513e411ae4c5b3d67d886807accdfddccbcc388d03e242e4597457d42783525e8ddcf8dbf9064799
-  data.tar.gz: 46df8a4e8dff1054f60b8545db22df35159c20ee0ab405d78795d476cc990efffd7d8cfe635f7641667d78cbac38fc0d5709ae8e802d040989356a883d3a4cd8
+  metadata.gz: 6a76055079c7bfa4a229fda0861f90276c94cb64859ae7ce3c269e86656f3ad69fb3d5e8ed7eb2b82e9fe1facb0a73619951f099aafbbac7b0b503a52eb21046
+  data.tar.gz: f541eec119d175cc50563bc2673d60dae6246a43bcf570f9fe4db9a6295697ea94f7ffddb709aa8c9d0264f274f48c42bca3322d3364c1edd47298a984421363

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 1.4.3 - 2020-06-10
+
+* Masking `password` parameter by default... doh.
+
+## 1.4.2 - 2020-02-09
+
+* Rollback of the previous two releases due to bugs introduced there.
+
+## 1.4.1 - 2020-02-08 - YANKED
+
+* [#91](https://github.com/trusche/httplog/pull/91) Fixed bug returning empty response with HTTP gem
+
+## 1.4.0 - 2020-01-19 - YANKED
+
+* [#85](https://github.com/trusche/httplog/pull/85) Parse JSON response and apply deep masking
+
+## 1.3.3 - 2019-11-14
+
+* [#83](https://github.com/trusche/httplog/pull/83) Support for graylog
+
+## 1.3.1 - 2019-06-07
+
+* [#76](https://github.com/trusche/httplog/pull/76) Added configurable logger method
+
 ## 1.3.0 - 2019-05-18
 
 * [#74](https://github.com/trusche/httplog/pull/74) Added ability to filter sensitive parameter values in the request (based on [#73](https://github.com/trusche/httplog/pull/73)). Default masking of `password` parameter

data/README.md

@@ -1,11 +1,14 @@
 ## httplog
 
+**VERSION 1.4.0 and 1.4.1 HAVE BEEN YANKED** from rubygems.org due to [this issue](https://github.com/trusche/httplog/issues/89), please update to version 1.4.2 or higher. Sorry about that...
+
 [![Gem Version](https://badge.fury.io/rb/httplog.svg)](http://badge.fury.io/rb/httplog) [![Build Status](https://travis-ci.org/trusche/httplog.svg?branch=master)](https://travis-ci.org/trusche/httplog) [![Code Climate](https://codeclimate.com/github/trusche/httplog.svg)](https://codeclimate.com/github/trusche/httplog)
 [![Release Version](https://img.shields.io/github/release/trusche/httplog.svg)](https://img.shields.io/github/release/trusche/httplog.svg)
+<a href="https://www.bearer.sh?ref=httplog"><img src="/bearer-badge.png" height="20px"/></a>
 
 Log outgoing HTTP requests made from your application. Helps with debugging pesky API error responses, or just generally understanding what's going on under the hood.
 
-Requires ruby >= 2.4.
+Requires ruby >= 2.5
 
 This gem works with the following ruby modules and libraries:
 
@@ -29,6 +32,10 @@ This is very much a development and debugging tool; it is **not recommended** to
 use this in a production environment as it is monkey-patching the respective HTTP implementations.
 You have been warned - use at your own risk.
 
+Httplog is kindly sponsored by <a href="https://www.bearer.sh?ref=httplog">Bearer.sh</a> - go check them out please!
+
+<a href="https://www.bearer.sh?ref=httplog"><img src="/bearer-sponsor.png" height="72px" /></a>
+
 ### Installation
 
     gem install httplog
@@ -54,8 +61,9 @@ HttpLog.configure do |config|
   # Enable or disable all logging
   config.enabled = true
 
-  # You can assign a different logger
+  # You can assign a different logger or method to call on that logger
   config.logger = Logger.new($stdout)
+  config.logger_method = :log
 
   # I really wouldn't change this...
   config.severity = Logger::Severity::DEBUG
@@ -82,7 +90,18 @@ HttpLog.configure do |config|
   config.url_whitelist_pattern = nil
   config.url_blacklist_pattern = nil
 
-  # Mask the values of sensitive requestparameters
+  # Mask sensitive information in request and response JSON data.
+  # Enable global JSON masking by setting the parameter to `/.*/`
+  config.url_masked_body_pattern = nil
+
+  # You can specify any custom JSON serializer that implements `load` and `dump` class methods
+  # to parse JSON responses
+  config.json_parser = JSON
+
+  # When using graylog, you can supply a formatter here - see below for details
+  config.graylog_formatter = nil
+
+  # Mask the values of sensitive request parameters
   config.filter_parameters = %w[password]
 end
 ```
@@ -97,6 +116,9 @@ HttpLog.configure do |config|
 end
 ```
 
+If you're running a (hopefully patched) legacy Rails 3 app, you may need to set
+`config.logger_method = :add` due to its somewhat unusual logger.
+
 You can colorize the output to make it stand out in your logfile, either with a single color
 for the text:
 
@@ -116,6 +138,37 @@ end
 
 For more color options please refer to the [rainbow documentation](https://github.com/sickill/rainbow)
 
+### Graylog logging
+
+If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT",
+you can use this configuration:
+
+```ruby
+FORMATTER = Lograge::Formatters::KeyValue.new
+
+HttpLog.configure do |config|
+  config.logger            = <your GELF::Logger>
+  config.logger_method     = :add
+  config.severity          = GELF::Levels::DEBUG
+  config.graylog_formatter = FORMATTER
+end
+```
+
+You also can use GELF Graylog format this way:
+
+```ruby
+class Lograge::Formatters::Graylog2HttpLog < Lograge::Formatters::Graylog2
+  def short_message data
+    data[:response_body] = data[:response_body].to_s.byteslice(0, 32_766) unless data[:response_body].blank?
+    "[httplog] [#{data[:response_code]}] #{data[:method]} #{data[:url]}"
+  end
+end
+
+FORMATTER = Lograge::Formatters::Graylog2HttpLog.new
+```
+
+Or define your own class that implements the `call` method
+
 ### Compact logging
 
 If the log is too noisy for you, but you don't want to completely disable it either, set the `compact_log` option to `true`. This will log each request in a single line with method, request URI, response status and time, but no data or headers. No need to disable any other options individually.
@@ -228,19 +281,3 @@ This will launch a simple rack server on port 9292 and run all tests locally aga
 If you have any issues with or feature requests for httplog,
 please [open an issue](https://github.com/trusche/httplog/issues) on GitHub
 or fork the project and send a pull request. **Please include passing specs with all pull requests.**
-
-### Contributors
-
-Thanks to these fine folks for contributing pull requests:
-
-* [Doug Johnston](https://github.com/dougjohnston)
-* [Eric Cohen](https://github.com/eirc)
-* [Nikos Dimitrakopoulos](https://github.com/nikosd)
-* [Marcos Hack](https://github.com/marcoshack)
-* [Andrew Hammond](https://github.com/andrhamm)
-* [Chris Keele](https://github.com/christhekeele)
-* [Ryan Souza](https://github.com/ryansouza)
-* [Ilya Bondarenko](https://github.com/sedx)
-* [Kostas Zacharakis](https://github.com/kzacharakis)
-* [Yuri Smirnov](https://github.com/tycooon)
-* [Manuel Bustillo Alonso](https://github.com/bustikiller)

data/httplog.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |gem|
   gem.description = "Log outgoing HTTP requests made from your application. Helpful for tracking API calls
                      of third party gems that don't provide their own log output."
 
-  gem.files         = `git ls-files`.split("\n")
+  gem.files         = Dir['lib/**/*.rb'] +
+                        %w(httplog.gemspec README.md CHANGELOG.md)
   gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.require_paths = ['lib']
 
@@ -27,7 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'excon', ['~> 0.60']
   gem.add_development_dependency 'faraday', ['~> 0.14']
   gem.add_development_dependency 'guard-rspec', ['~> 4.7']
-  gem.add_development_dependency 'http', ['~> 3.0']
+  gem.add_development_dependency 'http', ['~> 4.0']
   gem.add_development_dependency 'httparty', ['~> 0.16']
   gem.add_development_dependency 'httpclient', ['~> 2.8']
   gem.add_development_dependency 'listen', ['~> 3.0']
@@ -36,6 +37,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rspec', ['~> 3.7']
   gem.add_development_dependency 'simplecov', ['~> 0.15']
   gem.add_development_dependency 'thin', ['~> 1.7']
+  gem.add_development_dependency 'oj', ['>= 3.9.2']
 
   gem.add_dependency 'rack', ['>= 1.0']
   gem.add_dependency 'rainbow', ['>= 2.0.0']

data/lib/httplog/adapters/ethon.rb

@@ -39,7 +39,8 @@ if defined?(Ethon)
             response_headers: headers.map { |header| header.split(/:\s/) }.to_h,
             benchmark: bm,
             encoding: encoding,
-            content_type: content_type
+            content_type: content_type,
+            mask_body: HttpLog.masked_body_url?(url)
           )
           return_code
         end

data/lib/httplog/adapters/excon.rb

@@ -45,7 +45,8 @@ if defined?(Excon)
           response_headers: headers,
           benchmark: bm,
           encoding: headers['Content-Encoding'],
-          content_type: headers['Content-Type']
+          content_type: headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
         result
       end

data/lib/httplog/adapters/http.rb

@@ -12,7 +12,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
           @response = send(orig_request_method, req, options)
         end
 
-        if HttpLog.url_approved?(req.uri)
+        uri = req.uri
+        if HttpLog.url_approved?(uri)
           body = if defined?(::HTTP::Request::Body)
                    req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body)
                  else
@@ -21,7 +22,7 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
 
           HttpLog.call(
             method: req.verb,
-            url: req.uri,
+            url: uri,
             request_body: body,
             request_headers: req.headers,
             response_code: @response.code,
@@ -29,7 +30,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(uri)
           )
 
           body.rewind if body.respond_to?(:rewind)

data/lib/httplog/adapters/httpclient.rb

@@ -16,12 +16,13 @@ if defined?(::HTTPClient)
         end
       end
 
-      if HttpLog.url_approved?(req.header.request_uri)
+      request_uri = req.header.request_uri
+      if HttpLog.url_approved?(request_uri)
         res = conn.pop
 
         HttpLog.call(
           method: req.header.request_method,
-          url: req.header.request_uri,
+          url: request_uri,
           request_body: req.body,
           request_headers: req.headers,
           response_code: res.status_code,
@@ -29,7 +30,8 @@ if defined?(::HTTPClient)
           response_headers: res.headers,
           benchmark: bm,
           encoding: res.headers['Content-Encoding'],
-          content_type: res.headers['Content-Type']
+          content_type: res.headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(request_uri)
         )
         conn.push(res)
       end
@@ -41,7 +43,7 @@ if defined?(::HTTPClient)
       alias orig_create_socket create_socket
 
       # up to version 2.6, the method signature is `create_socket(site)`; after that,
-      # it's `create_socket(hort, port)`
+      # it's `create_socket(host, port)`
       if instance_method(:create_socket).arity == 1
         def create_socket(site)
           if HttpLog.url_approved?("#{site.host}:#{site.port}")

data/lib/httplog/adapters/net_http.rb

@@ -23,7 +23,8 @@ module Net
           response_headers: @response.each_header.collect,
           benchmark: bm,
           encoding: @response['Content-Encoding'],
-          content_type: @response['Content-Type']
+          content_type: @response['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
       end
 

data/lib/httplog/adapters/patron.rb

@@ -20,7 +20,8 @@ if defined?(Patron)
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(url)
           )
         end
 

data/lib/httplog/configuration.rb

@@ -5,7 +5,9 @@ module HttpLog
     attr_accessor :enabled,
                   :compact_log,
                   :json_log,
+                  :graylog_formatter,
                   :logger,
+                  :logger_method,
                   :severity,
                   :prefix,
                   :log_connect,
@@ -17,33 +19,39 @@ module HttpLog
                   :log_benchmark,
                   :url_whitelist_pattern,
                   :url_blacklist_pattern,
+                  :url_masked_body_pattern,
                   :color,
                   :prefix_data_lines,
                   :prefix_response_lines,
                   :prefix_line_numbers,
+                  :json_parser,
                   :filter_parameters
 
     def initialize
-      @enabled               = true
-      @compact_log           = false
-      @json_log              = false
-      @logger                = Logger.new($stdout)
-      @severity              = Logger::Severity::DEBUG
-      @prefix                = LOG_PREFIX
-      @log_connect           = true
-      @log_request           = true
-      @log_headers           = false
-      @log_data              = true
-      @log_status            = true
-      @log_response          = true
-      @log_benchmark         = true
-      @url_whitelist_pattern = nil
-      @url_blacklist_pattern = nil
-      @color                 = false
-      @prefix_data_lines     = false
-      @prefix_response_lines = false
-      @prefix_line_numbers   = false
-      @filter_parameters     = []
+      @enabled                 = true
+      @compact_log             = false
+      @json_log                = false
+      @graylog_formatter       = nil
+      @logger                  = Logger.new($stdout)
+      @logger_method           = :log
+      @severity                = Logger::Severity::DEBUG
+      @prefix                  = LOG_PREFIX
+      @log_connect             = true
+      @log_request             = true
+      @log_headers             = false
+      @log_data                = true
+      @log_status              = true
+      @log_response            = true
+      @log_benchmark           = true
+      @url_whitelist_pattern   = nil
+      @url_blacklist_pattern   = nil
+      @url_masked_body_pattern = nil
+      @color                   = false
+      @prefix_data_lines       = false
+      @prefix_response_lines   = false
+      @prefix_line_numbers     = false
+      @json_parser             = JSON
+      @filter_parameters       = %w[password]
     end
   end
 end

data/lib/httplog/http_log.rb

@@ -29,8 +29,11 @@ module HttpLog
     end
 
     def call(options = {})
+      parse_request(options)
       if config.json_log
         log_json(options)
+      elsif config.graylog_formatter
+        log_graylog(options)
       elsif config.compact_log
         log_compact(options[:method], options[:url], options[:response_code], options[:benchmark])
       else
@@ -40,7 +43,7 @@ module HttpLog
         HttpLog.log_status(options[:response_code])
         HttpLog.log_benchmark(options[:benchmark])
         HttpLog.log_headers(options[:response_headers])
-        HttpLog.log_body(options[:response_body], options[:encoding], options[:content_type])
+        HttpLog.log_body(options[:response_body], options[:mask_body], options[:encoding], options[:content_type])
       end
     end
 
@@ -50,10 +53,14 @@ module HttpLog
       !config.url_whitelist_pattern || url.to_s.match(config.url_whitelist_pattern)
     end
 
+    def masked_body_url?(url)
+      config.filter_parameters.any? && config.url_masked_body_pattern && url.to_s.match(config.url_masked_body_pattern)
+    end
+
     def log(msg)
       return unless config.enabled
 
-      config.logger.log(config.severity, colorize(prefix + msg))
+      config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg.to_s))
     end
 
     def log_connection(host, port = nil)
@@ -89,10 +96,10 @@ module HttpLog
       log("Benchmark: #{seconds.to_f.round(6)} seconds")
     end
 
-    def log_body(body, encoding = nil, content_type = nil)
+    def log_body(body, mask_body, encoding = nil, content_type = nil)
       return unless config.log_response
 
-      data = parse_body(body, encoding, content_type)
+      data = parse_body(body, mask_body, encoding, content_type)
 
       if config.prefix_response_lines
         log('Response:')
@@ -104,17 +111,19 @@ module HttpLog
       log("Response: #{e.message}")
     end
 
-    def parse_body(body, encoding, content_type)
-      unless text_based?(content_type)
-        raise BodyParsingError, "(not showing binary data)"
-      end
+    def parse_body(body, mask_body, encoding, content_type)
+      raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type)
+
 
       if body.is_a?(Net::ReadAdapter)
         # open-uri wraps the response in a Net::ReadAdapter that defers reading
-        # the content, so the reponse body is not available here.
+        # the content, so the response body is not available here.
         raise BodyParsingError, '(not available yet)'
       end
 
+      body = body.to_s if defined?(HTTP::Response::Body) && body.is_a?(HTTP::Response::Body)
+      body = body.dup
+
       if encoding =~ /gzip/ && body && !body.empty?
         begin
           sio = StringIO.new(body.to_s)
@@ -125,14 +134,25 @@ module HttpLog
         end
       end
 
-      utf_encoded(body.to_s, content_type)
+      result = utf_encoded(body.to_s, content_type)
+
+      if mask_body && body && !body.empty?
+        if content_type =~ /json/
+          result = begin
+                     masked_data config.json_parser.load(result)
+                   rescue => e
+                     'Failed to mask response body: ' + e.message
+                   end
+        else
+          result = masked(result)
+        end
+      end
+      result
     end
 
     def log_data(data)
       return unless config.log_data
 
-      data = utf_encoded(masked(data.dup).to_s) unless data.nil?
-
       if config.prefix_data_lines
         log('Data:')
         log_data_lines(data)
@@ -147,38 +167,6 @@ module HttpLog
       log("#{method.to_s.upcase} #{masked(uri)} completed with status code #{status} in #{seconds.to_f.round(6)} seconds")
     end
 
-    def log_json(data = {})
-      return unless config.json_log
-
-      data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol)
-
-      parsed_body = begin
-        parse_body(data[:response_body], data[:encoding], data[:content_type])
-      rescue BodyParsingError => e
-        e.message
-      end
-
-      if config.compact_log
-        log({
-          method: data[:method].to_s.upcase,
-          url: masked(data[:url]),
-          response_code: data[:response_code].to_i,
-          benchmark: data[:benchmark]
-        }.to_json)
-      else
-        log({
-          method: data[:method].to_s.upcase,
-          url: masked(data[:url]),
-          request_body: masked(data[:request_body]),
-          request_headers: masked(data[:request_headers].to_h),
-          response_code: data[:response_code].to_i,
-          response_body: parsed_body,
-          response_headers: data[:response_headers].to_h,
-          benchmark: data[:benchmark]
-        }.to_json)
-      end
-    end
-
     def transform_response_code(response_code_name)
       Rack::Utils::HTTP_STATUS_CODES.detect { |_k, v| v.to_s.casecmp(response_code_name.to_s).zero? }.first
     end
@@ -199,6 +187,70 @@ module HttpLog
 
     private
 
+    def log_json(data = {})
+      return unless config.json_log
+
+      log(
+        begin
+          dump_json(data)
+        rescue
+          data[:response_body] = "#{config.json_parser} dump failed"
+          data[:request_body]  = "#{config.json_parser} dump failed"
+          dump_json(data)
+        end
+      )
+    end
+
+    def dump_json(data)
+      config.json_parser.dump(json_payload(data))
+    end
+
+    def log_graylog(data)
+      result = json_payload(data)
+      begin
+        send_to_graylog result
+      rescue
+        result[:response_body] = 'Graylog JSON dump failed'
+        result[:request_body]  = 'Graylog JSON dump failed'
+        send_to_graylog result
+      end
+    end
+
+    def send_to_graylog data
+      data.compact!
+      config.logger.public_send(config.logger_method, config.severity, config.graylog_formatter.call(data))
+    end
+
+    def json_payload(data = {})
+      data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol)
+
+      parsed_body = begin
+                      parse_body(data[:response_body], data[:mask_body], data[:encoding], data[:content_type])
+                    rescue BodyParsingError => e
+                      e.message
+                    end
+
+      if config.compact_log
+        {
+          method:        data[:method].to_s.upcase,
+          url:           masked(data[:url]),
+          response_code: data[:response_code].to_i,
+          benchmark:     data[:benchmark]
+        }
+      else
+        {
+          method:           data[:method].to_s.upcase,
+          url:              masked(data[:url]),
+          request_body:     data[:request_body],
+          request_headers:  masked(data[:request_headers].to_h),
+          response_code:    data[:response_code].to_i,
+          response_body:    parsed_body,
+          response_headers: data[:response_headers].to_h,
+          benchmark:        data[:benchmark]
+        }
+      end
+    end
+
     def masked(msg, key=nil)
       return msg if config.filter_parameters.empty?
       return msg if msg.nil?
@@ -211,7 +263,8 @@ module HttpLog
       case msg
       when *string_classes
         config.filter_parameters.reduce(msg) do |m,key|
-          m.to_s.gsub(/(#{key})=[^&]+/i, "#{key}=#{PARAM_MASK}")
+          scrubbed = m.to_s.encode('UTF-8', invalid: :replace, undef: :replace)
+          scrubbed.gsub(/(#{key})=[^&]+/i, "#{key}=#{PARAM_MASK}")
         end
       # ...and recurse over hashes
       when *hash_classes
@@ -222,6 +275,38 @@ module HttpLog
       end
     end
 
+    def parse_request(options)
+      return if options[:request_body].nil?
+
+      # Downcase content-type and content-encoding because ::HTTP returns "Content-Type" and "Content-Encoding"
+      headers = options[:request_headers].find_all do |header, _|
+        %w[content-type Content-Type content-encoding Content-Encoding].include? header
+      end.to_h.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
+
+      copy = options[:request_body].dup
+
+      options[:request_body] = if text_based?(headers['content-type']) && options[:mask_body]
+                                 begin
+                                   parse_body(copy, options[:mask_body], headers['content-encoding'], headers['content-type'])
+                                 rescue BodyParsingError => e
+                                   log(e.message)
+                                 end
+                               else
+                                 masked(copy).to_s
+                               end
+    end
+
+    def masked_data msg
+      case msg
+      when Hash
+        Hash[msg.map { |k, v| [k, config.filter_parameters.include?(k.downcase) ? PARAM_MASK : masked_data(v)] }]
+      when Array
+        msg.map { |element| masked_data(element) }
+      else
+        msg
+      end
+    end
+
     def string_classes
       @string_classes ||= begin
         string_classes = [String]