httplog 1.3.3 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 899f0ffc584e0b7c88daa0e6895502f4d1a20d850d59ae1483de9dac829346f3
-  data.tar.gz: ee045f1978c1955ef213382b166bcf6e824fe42c352704476b3683ef0c495f54
+  metadata.gz: 5332ba82035e300b5010a5cbcb125d9c90e083a405c43d1be8df1d626fc40fe1
+  data.tar.gz: 9692d53da3306a12bb14128f8365a9d7524fe6d443474c3dbdc11c21a8f3f6e9
 SHA512:
-  metadata.gz: a7a9ae16ba3ff2391021c8284d4e0ff43c40b30bfed7f33d8f288312410a527dec550af3eb993c355e8e6d07132fa46378b09dc6535ffdea00650d256e34c9c6
-  data.tar.gz: 58f22a5832993eefef75b2a2a3f9e6ab6c41286553b404b44e2df8fbee3a6623116b7729222a3ef842380969ea8edc53b9dd4b93755b612594b1f473560d2b3f
+  metadata.gz: 7829d7f38262b59ac69ae08e638aad00315322025fed502abf50e4ca468ea4286473c54984277ec98d436ff04fc86157ff38e5e529ec6ee8a9bef5a362f80e48
+  data.tar.gz: 4d0ccb0d024d21de6dcc9d116353810cf2e8adbc936c9b9ecbd38887931851555c19eb2756b7f43f298b9f7d6cc855322d75f4c473d813c35cc574c63cdb9ae0

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.4.1 - 2020-02-08 - YANKED
+
+* [#91](https://github.com/trusche/httplog/pull/91) Fixed bug returning empty response with HTTP gem
+
+## 1.4.0 - 2020-01-19 - YANKED
+
+* [#85](https://github.com/trusche/httplog/pull/85) Parse JSON response and apply deep masking
+
 ## 1.3.3 - 2019-11-14
 
 * [#83](https://github.com/trusche/httplog/pull/83) Support for graylog

data/README.md

@@ -1,5 +1,7 @@
 ## httplog
 
+**VERSION 1.4.0 and 1.4.1 HAVE BEEN YANKED** from rubygems.org due to [this issue](https://github.com/trusche/httplog/issues/89), please update to version 1.4.2. Sorry about that...
+
 [![Gem Version](https://badge.fury.io/rb/httplog.svg)](http://badge.fury.io/rb/httplog) [![Build Status](https://travis-ci.org/trusche/httplog.svg?branch=master)](https://travis-ci.org/trusche/httplog) [![Code Climate](https://codeclimate.com/github/trusche/httplog.svg)](https://codeclimate.com/github/trusche/httplog)
 [![Release Version](https://img.shields.io/github/release/trusche/httplog.svg)](https://img.shields.io/github/release/trusche/httplog.svg)
 
@@ -86,7 +88,15 @@ HttpLog.configure do |config|
   config.url_whitelist_pattern = nil
   config.url_blacklist_pattern = nil
 
-  # Mask the values of sensitive requestparameters
+  # Mask sensitive information in request and response JSON data.
+  # Enable global JSON masking by setting the parameter to `/.*/`
+  config.url_masked_body_pattern = nil
+
+  # You can specify any custom JSON serializer that implements `load` and `dump` class methods
+  # to parse JSON responses
+  config.json_parser = JSON
+
+  # Mask the values of sensitive request parameters
   config.filter_parameters = %w[password]
 end
 ```
@@ -121,7 +131,7 @@ HttpLog.configure do |config|
 end
 ```
 
-If you use Graylog and want to use its search features such as "rounded_benchmark:>1 AND method:PUT",
+If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT",
 you can use this configuration:
 
 ```ruby

data/httplog.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rspec', ['~> 3.7']
   gem.add_development_dependency 'simplecov', ['~> 0.15']
   gem.add_development_dependency 'thin', ['~> 1.7']
+  gem.add_development_dependency 'oj', ['>= 3.9.2']
 
   gem.add_dependency 'rack', ['>= 1.0']
   gem.add_dependency 'rainbow', ['>= 2.0.0']

data/lib/httplog/adapters/ethon.rb

@@ -39,7 +39,8 @@ if defined?(Ethon)
             response_headers: headers.map { |header| header.split(/:\s/) }.to_h,
             benchmark: bm,
             encoding: encoding,
-            content_type: content_type
+            content_type: content_type,
+            mask_body: HttpLog.masked_body_url?(url)
           )
           return_code
         end

data/lib/httplog/adapters/excon.rb

@@ -45,7 +45,8 @@ if defined?(Excon)
           response_headers: headers,
           benchmark: bm,
           encoding: headers['Content-Encoding'],
-          content_type: headers['Content-Type']
+          content_type: headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
         result
       end

data/lib/httplog/adapters/http.rb

@@ -12,7 +12,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
           @response = send(orig_request_method, req, options)
         end
 
-        if HttpLog.url_approved?(req.uri)
+        uri = req.uri
+        if HttpLog.url_approved?(uri)
           body = if defined?(::HTTP::Request::Body)
                    req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body)
                  else
@@ -21,7 +22,7 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
 
           HttpLog.call(
             method: req.verb,
-            url: req.uri,
+            url: uri,
             request_body: body,
             request_headers: req.headers,
             response_code: @response.code,
@@ -29,7 +30,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(uri)
           )
 
           body.rewind if body.respond_to?(:rewind)

data/lib/httplog/adapters/httpclient.rb

@@ -16,12 +16,13 @@ if defined?(::HTTPClient)
         end
       end
 
-      if HttpLog.url_approved?(req.header.request_uri)
+      request_uri = req.header.request_uri
+      if HttpLog.url_approved?(request_uri)
         res = conn.pop
 
         HttpLog.call(
           method: req.header.request_method,
-          url: req.header.request_uri,
+          url: request_uri,
           request_body: req.body,
           request_headers: req.headers,
           response_code: res.status_code,
@@ -29,7 +30,8 @@ if defined?(::HTTPClient)
           response_headers: res.headers,
           benchmark: bm,
           encoding: res.headers['Content-Encoding'],
-          content_type: res.headers['Content-Type']
+          content_type: res.headers['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(request_uri)
         )
         conn.push(res)
       end
@@ -41,7 +43,7 @@ if defined?(::HTTPClient)
       alias orig_create_socket create_socket
 
       # up to version 2.6, the method signature is `create_socket(site)`; after that,
-      # it's `create_socket(hort, port)`
+      # it's `create_socket(host, port)`
       if instance_method(:create_socket).arity == 1
         def create_socket(site)
           if HttpLog.url_approved?("#{site.host}:#{site.port}")

data/lib/httplog/adapters/net_http.rb

@@ -23,7 +23,8 @@ module Net
           response_headers: @response.each_header.collect,
           benchmark: bm,
           encoding: @response['Content-Encoding'],
-          content_type: @response['Content-Type']
+          content_type: @response['Content-Type'],
+          mask_body: HttpLog.masked_body_url?(url)
         )
       end
 

data/lib/httplog/adapters/patron.rb

@@ -20,7 +20,8 @@ if defined?(Patron)
             response_headers: @response.headers,
             benchmark: bm,
             encoding: @response.headers['Content-Encoding'],
-            content_type: @response.headers['Content-Type']
+            content_type: @response.headers['Content-Type'],
+            mask_body: HttpLog.masked_body_url?(url)
           )
         end
 

data/lib/httplog/configuration.rb

@@ -19,35 +19,39 @@ module HttpLog
                   :log_benchmark,
                   :url_whitelist_pattern,
                   :url_blacklist_pattern,
+                  :url_masked_body_pattern,
                   :color,
                   :prefix_data_lines,
                   :prefix_response_lines,
                   :prefix_line_numbers,
+                  :json_parser,
                   :filter_parameters
 
     def initialize
-      @enabled               = true
-      @compact_log           = false
-      @json_log              = false
-      @graylog               = false
-      @logger                = Logger.new($stdout)
-      @logger_method         = :log
-      @severity              = Logger::Severity::DEBUG
-      @prefix                = LOG_PREFIX
-      @log_connect           = true
-      @log_request           = true
-      @log_headers           = false
-      @log_data              = true
-      @log_status            = true
-      @log_response          = true
-      @log_benchmark         = true
-      @url_whitelist_pattern = nil
-      @url_blacklist_pattern = nil
-      @color                 = false
-      @prefix_data_lines     = false
-      @prefix_response_lines = false
-      @prefix_line_numbers   = false
-      @filter_parameters     = []
+      @enabled                 = true
+      @compact_log             = false
+      @json_log                = false
+      @graylog                 = false
+      @logger                  = Logger.new($stdout)
+      @logger_method           = :log
+      @severity                = Logger::Severity::DEBUG
+      @prefix                  = LOG_PREFIX
+      @log_connect             = true
+      @log_request             = true
+      @log_headers             = false
+      @log_data                = true
+      @log_status              = true
+      @log_response            = true
+      @log_benchmark           = true
+      @url_whitelist_pattern   = nil
+      @url_blacklist_pattern   = nil
+      @url_masked_body_pattern = nil
+      @color                   = false
+      @prefix_data_lines       = false
+      @prefix_response_lines   = false
+      @prefix_line_numbers     = false
+      @json_parser             = JSON
+      @filter_parameters       = []
     end
   end
 end

data/lib/httplog/http_log.rb

@@ -29,6 +29,7 @@ module HttpLog
     end
 
     def call(options = {})
+      parse_request(options)
       if config.json_log
         log_json(options)
       elsif config.graylog
@@ -42,7 +43,7 @@ module HttpLog
         HttpLog.log_status(options[:response_code])
         HttpLog.log_benchmark(options[:benchmark])
         HttpLog.log_headers(options[:response_headers])
-        HttpLog.log_body(options[:response_body], options[:encoding], options[:content_type])
+        HttpLog.log_body(options[:response_body], options[:mask_body], options[:encoding], options[:content_type])
       end
     end
 
@@ -52,10 +53,14 @@ module HttpLog
       !config.url_whitelist_pattern || url.to_s.match(config.url_whitelist_pattern)
     end
 
+    def masked_body_url?(url)
+      config.filter_parameters.any? && config.url_masked_body_pattern && url.to_s.match(config.url_masked_body_pattern)
+    end
+
     def log(msg)
       return unless config.enabled
 
-      config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg))
+      config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg.to_s))
     end
 
     def log_connection(host, port = nil)
@@ -91,10 +96,10 @@ module HttpLog
       log("Benchmark: #{seconds.to_f.round(6)} seconds")
     end
 
-    def log_body(body, encoding = nil, content_type = nil)
+    def log_body(body, mask_body, encoding = nil, content_type = nil)
       return unless config.log_response
 
-      data = parse_body(body, encoding, content_type)
+      data = parse_body(body, mask_body, encoding, content_type)
 
       if config.prefix_response_lines
         log('Response:')
@@ -106,10 +111,9 @@ module HttpLog
       log("Response: #{e.message}")
     end
 
-    def parse_body(body, encoding, content_type)
-      unless text_based?(content_type)
-        raise BodyParsingError, "(not showing binary data)"
-      end
+    def parse_body(body, mask_body, encoding, content_type)
+      raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type)
+
 
       if body.is_a?(Net::ReadAdapter)
         # open-uri wraps the response in a Net::ReadAdapter that defers reading
@@ -117,6 +121,9 @@ module HttpLog
         raise BodyParsingError, '(not available yet)'
       end
 
+      body = body.to_s if defined?(HTTP::Response::Body) && body.is_a?(HTTP::Response::Body)
+      body = body.dup
+
       if encoding =~ /gzip/ && body && !body.empty?
         begin
           sio = StringIO.new(body.to_s)
@@ -127,14 +134,25 @@ module HttpLog
         end
       end
 
-      utf_encoded(body.to_s, content_type)
+      result = utf_encoded(body.to_s, content_type)
+
+      if mask_body && body && !body.empty?
+        if content_type =~ /json/
+          result = begin
+                     masked_data config.json_parser.load(result)
+                   rescue => e
+                     'Failed to mask response body: ' + e.message
+                   end
+        else
+          result = masked(result)
+        end
+      end
+      result
     end
 
     def log_data(data)
       return unless config.log_data
 
-      data = utf_encoded(masked(data.dup).to_s) unless data.nil?
-
       if config.prefix_data_lines
         log('Data:')
         log_data_lines(data)
@@ -172,22 +190,43 @@ module HttpLog
     def log_json(data = {})
       return unless config.json_log
 
-      log(json_payload(data).to_json)
+      log(
+        begin
+          dump_json(data)
+        rescue
+          data[:response_body] = "#{config.json_parser} dump failed"
+          data[:request_body]  = "#{config.json_parser} dump failed"
+          dump_json(data)
+        end
+      )
+    end
+
+    def dump_json(data)
+      config.json_parser.dump(json_payload(data))
     end
 
-    def log_graylog(data = {})
+    def log_graylog(data)
       result = json_payload(data)
 
-      result[:rounded_benchmark] = data[:benchmark].round
-      result[:short_message]     = result.delete(:url)
-      config.logger.public_send(config.logger_method, config.severity, result)
+      result[:short_message] = result.delete(:url)
+      begin
+        send_to_graylog result
+      rescue
+        result[:response_body] = 'Graylog JSON dump failed'
+        result[:request_body]  = 'Graylog JSON dump failed'
+        send_to_graylog result
+      end
+    end
+
+    def send_to_graylog data
+      config.logger.public_send(config.logger_method, config.severity, data)
     end
 
     def json_payload(data = {})
       data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol)
 
       parsed_body = begin
-                      parse_body(data[:response_body], data[:encoding], data[:content_type])
+                      parse_body(data[:response_body], data[:mask_body], data[:encoding], data[:content_type])
                     rescue BodyParsingError => e
                       e.message
                     end
@@ -203,7 +242,7 @@ module HttpLog
         {
           method:           data[:method].to_s.upcase,
           url:              masked(data[:url]),
-          request_body:     masked(data[:request_body]),
+          request_body:     data[:request_body],
           request_headers:  masked(data[:request_headers].to_h),
           response_code:    data[:response_code].to_i,
           response_body:    parsed_body,
@@ -236,6 +275,38 @@ module HttpLog
       end
     end
 
+    def parse_request(options)
+      return if options[:request_body].nil?
+
+      # Downcase content-type and content-encoding because ::HTTP returns "Content-Type" and "Content-Encoding"
+      headers = options[:request_headers].find_all do |header, _|
+        %w[content-type Content-Type content-encoding Content-Encoding].include? header
+      end.to_h.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
+
+      copy = options[:request_body].dup
+
+      options[:request_body] = if text_based?(headers['content-type']) && options[:mask_body]
+                                 begin
+                                   parse_body(copy, options[:mask_body], headers['content-encoding'], headers['content-type'])
+                                 rescue BodyParsingError => e
+                                   log(e.message)
+                                 end
+                               else
+                                 masked(copy).to_s
+                               end
+    end
+
+    def masked_data msg
+      case msg
+      when Hash
+        Hash[msg.map { |k, v| [k, config.filter_parameters.include?(k.downcase) ? PARAM_MASK : masked_data(v)] }]
+      when Array
+        msg.map { |element| masked_data(element) }
+      else
+        msg
+      end
+    end
+
     def string_classes
       @string_classes ||= begin
         string_classes = [String]

data/lib/httplog/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HttpLog
-  VERSION = '1.3.3'.freeze
+  VERSION = '1.4.2'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httplog
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.4.2
 platform: ruby
 authors:
 - Thilo Rusche
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2020-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethon
@@ -192,6 +192,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.9.2
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -260,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Log outgoing HTTP requests.