httplog 1.3.3 → 1.4.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 899f0ffc584e0b7c88daa0e6895502f4d1a20d850d59ae1483de9dac829346f3
4
- data.tar.gz: ee045f1978c1955ef213382b166bcf6e824fe42c352704476b3683ef0c495f54
3
+ metadata.gz: 5332ba82035e300b5010a5cbcb125d9c90e083a405c43d1be8df1d626fc40fe1
4
+ data.tar.gz: 9692d53da3306a12bb14128f8365a9d7524fe6d443474c3dbdc11c21a8f3f6e9
5
5
  SHA512:
6
- metadata.gz: a7a9ae16ba3ff2391021c8284d4e0ff43c40b30bfed7f33d8f288312410a527dec550af3eb993c355e8e6d07132fa46378b09dc6535ffdea00650d256e34c9c6
7
- data.tar.gz: 58f22a5832993eefef75b2a2a3f9e6ab6c41286553b404b44e2df8fbee3a6623116b7729222a3ef842380969ea8edc53b9dd4b93755b612594b1f473560d2b3f
6
+ metadata.gz: 7829d7f38262b59ac69ae08e638aad00315322025fed502abf50e4ca468ea4286473c54984277ec98d436ff04fc86157ff38e5e529ec6ee8a9bef5a362f80e48
7
+ data.tar.gz: 4d0ccb0d024d21de6dcc9d116353810cf2e8adbc936c9b9ecbd38887931851555c19eb2756b7f43f298b9f7d6cc855322d75f4c473d813c35cc574c63cdb9ae0
@@ -1,3 +1,11 @@
1
+ ## 1.4.1 - 2020-02-08 - YANKED
2
+
3
+ * [#91](https://github.com/trusche/httplog/pull/91) Fixed bug returning empty response with HTTP gem
4
+
5
+ ## 1.4.0 - 2020-01-19 - YANKED
6
+
7
+ * [#85](https://github.com/trusche/httplog/pull/85) Parse JSON response and apply deep masking
8
+
1
9
  ## 1.3.3 - 2019-11-14
2
10
 
3
11
  * [#83](https://github.com/trusche/httplog/pull/83) Support for graylog
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  ## httplog
2
2
 
3
+ **VERSION 1.4.0 and 1.4.1 HAVE BEEN YANKED** from rubygems.org due to [this issue](https://github.com/trusche/httplog/issues/89), please update to version 1.4.2. Sorry about that...
4
+
3
5
  [![Gem Version](https://badge.fury.io/rb/httplog.svg)](http://badge.fury.io/rb/httplog) [![Build Status](https://travis-ci.org/trusche/httplog.svg?branch=master)](https://travis-ci.org/trusche/httplog) [![Code Climate](https://codeclimate.com/github/trusche/httplog.svg)](https://codeclimate.com/github/trusche/httplog)
4
6
  [![Release Version](https://img.shields.io/github/release/trusche/httplog.svg)](https://img.shields.io/github/release/trusche/httplog.svg)
5
7
 
@@ -86,7 +88,15 @@ HttpLog.configure do |config|
86
88
  config.url_whitelist_pattern = nil
87
89
  config.url_blacklist_pattern = nil
88
90
 
89
- # Mask the values of sensitive requestparameters
91
+ # Mask sensitive information in request and response JSON data.
92
+ # Enable global JSON masking by setting the parameter to `/.*/`
93
+ config.url_masked_body_pattern = nil
94
+
95
+ # You can specify any custom JSON serializer that implements `load` and `dump` class methods
96
+ # to parse JSON responses
97
+ config.json_parser = JSON
98
+
99
+ # Mask the values of sensitive request parameters
90
100
  config.filter_parameters = %w[password]
91
101
  end
92
102
  ```
@@ -121,7 +131,7 @@ HttpLog.configure do |config|
121
131
  end
122
132
  ```
123
133
 
124
- If you use Graylog and want to use its search features such as "rounded_benchmark:>1 AND method:PUT",
134
+ If you use Graylog and want to use its search features such as "benchmark:>1 AND method:PUT",
125
135
  you can use this configuration:
126
136
 
127
137
  ```ruby
@@ -37,6 +37,7 @@ Gem::Specification.new do |gem|
37
37
  gem.add_development_dependency 'rspec', ['~> 3.7']
38
38
  gem.add_development_dependency 'simplecov', ['~> 0.15']
39
39
  gem.add_development_dependency 'thin', ['~> 1.7']
40
+ gem.add_development_dependency 'oj', ['>= 3.9.2']
40
41
 
41
42
  gem.add_dependency 'rack', ['>= 1.0']
42
43
  gem.add_dependency 'rainbow', ['>= 2.0.0']
@@ -39,7 +39,8 @@ if defined?(Ethon)
39
39
  response_headers: headers.map { |header| header.split(/:\s/) }.to_h,
40
40
  benchmark: bm,
41
41
  encoding: encoding,
42
- content_type: content_type
42
+ content_type: content_type,
43
+ mask_body: HttpLog.masked_body_url?(url)
43
44
  )
44
45
  return_code
45
46
  end
@@ -45,7 +45,8 @@ if defined?(Excon)
45
45
  response_headers: headers,
46
46
  benchmark: bm,
47
47
  encoding: headers['Content-Encoding'],
48
- content_type: headers['Content-Type']
48
+ content_type: headers['Content-Type'],
49
+ mask_body: HttpLog.masked_body_url?(url)
49
50
  )
50
51
  result
51
52
  end
@@ -12,7 +12,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
12
12
  @response = send(orig_request_method, req, options)
13
13
  end
14
14
 
15
- if HttpLog.url_approved?(req.uri)
15
+ uri = req.uri
16
+ if HttpLog.url_approved?(uri)
16
17
  body = if defined?(::HTTP::Request::Body)
17
18
  req.body.respond_to?(:source) ? req.body.source : req.body.instance_variable_get(:@body)
18
19
  else
@@ -21,7 +22,7 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
21
22
 
22
23
  HttpLog.call(
23
24
  method: req.verb,
24
- url: req.uri,
25
+ url: uri,
25
26
  request_body: body,
26
27
  request_headers: req.headers,
27
28
  response_code: @response.code,
@@ -29,7 +30,8 @@ if defined?(::HTTP::Client) && defined?(::HTTP::Connection)
29
30
  response_headers: @response.headers,
30
31
  benchmark: bm,
31
32
  encoding: @response.headers['Content-Encoding'],
32
- content_type: @response.headers['Content-Type']
33
+ content_type: @response.headers['Content-Type'],
34
+ mask_body: HttpLog.masked_body_url?(uri)
33
35
  )
34
36
 
35
37
  body.rewind if body.respond_to?(:rewind)
@@ -16,12 +16,13 @@ if defined?(::HTTPClient)
16
16
  end
17
17
  end
18
18
 
19
- if HttpLog.url_approved?(req.header.request_uri)
19
+ request_uri = req.header.request_uri
20
+ if HttpLog.url_approved?(request_uri)
20
21
  res = conn.pop
21
22
 
22
23
  HttpLog.call(
23
24
  method: req.header.request_method,
24
- url: req.header.request_uri,
25
+ url: request_uri,
25
26
  request_body: req.body,
26
27
  request_headers: req.headers,
27
28
  response_code: res.status_code,
@@ -29,7 +30,8 @@ if defined?(::HTTPClient)
29
30
  response_headers: res.headers,
30
31
  benchmark: bm,
31
32
  encoding: res.headers['Content-Encoding'],
32
- content_type: res.headers['Content-Type']
33
+ content_type: res.headers['Content-Type'],
34
+ mask_body: HttpLog.masked_body_url?(request_uri)
33
35
  )
34
36
  conn.push(res)
35
37
  end
@@ -41,7 +43,7 @@ if defined?(::HTTPClient)
41
43
  alias orig_create_socket create_socket
42
44
 
43
45
  # up to version 2.6, the method signature is `create_socket(site)`; after that,
44
- # it's `create_socket(hort, port)`
46
+ # it's `create_socket(host, port)`
45
47
  if instance_method(:create_socket).arity == 1
46
48
  def create_socket(site)
47
49
  if HttpLog.url_approved?("#{site.host}:#{site.port}")
@@ -23,7 +23,8 @@ module Net
23
23
  response_headers: @response.each_header.collect,
24
24
  benchmark: bm,
25
25
  encoding: @response['Content-Encoding'],
26
- content_type: @response['Content-Type']
26
+ content_type: @response['Content-Type'],
27
+ mask_body: HttpLog.masked_body_url?(url)
27
28
  )
28
29
  end
29
30
 
@@ -20,7 +20,8 @@ if defined?(Patron)
20
20
  response_headers: @response.headers,
21
21
  benchmark: bm,
22
22
  encoding: @response.headers['Content-Encoding'],
23
- content_type: @response.headers['Content-Type']
23
+ content_type: @response.headers['Content-Type'],
24
+ mask_body: HttpLog.masked_body_url?(url)
24
25
  )
25
26
  end
26
27
 
@@ -19,35 +19,39 @@ module HttpLog
19
19
  :log_benchmark,
20
20
  :url_whitelist_pattern,
21
21
  :url_blacklist_pattern,
22
+ :url_masked_body_pattern,
22
23
  :color,
23
24
  :prefix_data_lines,
24
25
  :prefix_response_lines,
25
26
  :prefix_line_numbers,
27
+ :json_parser,
26
28
  :filter_parameters
27
29
 
28
30
  def initialize
29
- @enabled = true
30
- @compact_log = false
31
- @json_log = false
32
- @graylog = false
33
- @logger = Logger.new($stdout)
34
- @logger_method = :log
35
- @severity = Logger::Severity::DEBUG
36
- @prefix = LOG_PREFIX
37
- @log_connect = true
38
- @log_request = true
39
- @log_headers = false
40
- @log_data = true
41
- @log_status = true
42
- @log_response = true
43
- @log_benchmark = true
44
- @url_whitelist_pattern = nil
45
- @url_blacklist_pattern = nil
46
- @color = false
47
- @prefix_data_lines = false
48
- @prefix_response_lines = false
49
- @prefix_line_numbers = false
50
- @filter_parameters = []
31
+ @enabled = true
32
+ @compact_log = false
33
+ @json_log = false
34
+ @graylog = false
35
+ @logger = Logger.new($stdout)
36
+ @logger_method = :log
37
+ @severity = Logger::Severity::DEBUG
38
+ @prefix = LOG_PREFIX
39
+ @log_connect = true
40
+ @log_request = true
41
+ @log_headers = false
42
+ @log_data = true
43
+ @log_status = true
44
+ @log_response = true
45
+ @log_benchmark = true
46
+ @url_whitelist_pattern = nil
47
+ @url_blacklist_pattern = nil
48
+ @url_masked_body_pattern = nil
49
+ @color = false
50
+ @prefix_data_lines = false
51
+ @prefix_response_lines = false
52
+ @prefix_line_numbers = false
53
+ @json_parser = JSON
54
+ @filter_parameters = []
51
55
  end
52
56
  end
53
57
  end
@@ -29,6 +29,7 @@ module HttpLog
29
29
  end
30
30
 
31
31
  def call(options = {})
32
+ parse_request(options)
32
33
  if config.json_log
33
34
  log_json(options)
34
35
  elsif config.graylog
@@ -42,7 +43,7 @@ module HttpLog
42
43
  HttpLog.log_status(options[:response_code])
43
44
  HttpLog.log_benchmark(options[:benchmark])
44
45
  HttpLog.log_headers(options[:response_headers])
45
- HttpLog.log_body(options[:response_body], options[:encoding], options[:content_type])
46
+ HttpLog.log_body(options[:response_body], options[:mask_body], options[:encoding], options[:content_type])
46
47
  end
47
48
  end
48
49
 
@@ -52,10 +53,14 @@ module HttpLog
52
53
  !config.url_whitelist_pattern || url.to_s.match(config.url_whitelist_pattern)
53
54
  end
54
55
 
56
+ def masked_body_url?(url)
57
+ config.filter_parameters.any? && config.url_masked_body_pattern && url.to_s.match(config.url_masked_body_pattern)
58
+ end
59
+
55
60
  def log(msg)
56
61
  return unless config.enabled
57
62
 
58
- config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg))
63
+ config.logger.public_send(config.logger_method, config.severity, colorize(prefix + msg.to_s))
59
64
  end
60
65
 
61
66
  def log_connection(host, port = nil)
@@ -91,10 +96,10 @@ module HttpLog
91
96
  log("Benchmark: #{seconds.to_f.round(6)} seconds")
92
97
  end
93
98
 
94
- def log_body(body, encoding = nil, content_type = nil)
99
+ def log_body(body, mask_body, encoding = nil, content_type = nil)
95
100
  return unless config.log_response
96
101
 
97
- data = parse_body(body, encoding, content_type)
102
+ data = parse_body(body, mask_body, encoding, content_type)
98
103
 
99
104
  if config.prefix_response_lines
100
105
  log('Response:')
@@ -106,10 +111,9 @@ module HttpLog
106
111
  log("Response: #{e.message}")
107
112
  end
108
113
 
109
- def parse_body(body, encoding, content_type)
110
- unless text_based?(content_type)
111
- raise BodyParsingError, "(not showing binary data)"
112
- end
114
+ def parse_body(body, mask_body, encoding, content_type)
115
+ raise BodyParsingError, "(not showing binary data)" unless text_based?(content_type)
116
+
113
117
 
114
118
  if body.is_a?(Net::ReadAdapter)
115
119
  # open-uri wraps the response in a Net::ReadAdapter that defers reading
@@ -117,6 +121,9 @@ module HttpLog
117
121
  raise BodyParsingError, '(not available yet)'
118
122
  end
119
123
 
124
+ body = body.to_s if defined?(HTTP::Response::Body) && body.is_a?(HTTP::Response::Body)
125
+ body = body.dup
126
+
120
127
  if encoding =~ /gzip/ && body && !body.empty?
121
128
  begin
122
129
  sio = StringIO.new(body.to_s)
@@ -127,14 +134,25 @@ module HttpLog
127
134
  end
128
135
  end
129
136
 
130
- utf_encoded(body.to_s, content_type)
137
+ result = utf_encoded(body.to_s, content_type)
138
+
139
+ if mask_body && body && !body.empty?
140
+ if content_type =~ /json/
141
+ result = begin
142
+ masked_data config.json_parser.load(result)
143
+ rescue => e
144
+ 'Failed to mask response body: ' + e.message
145
+ end
146
+ else
147
+ result = masked(result)
148
+ end
149
+ end
150
+ result
131
151
  end
132
152
 
133
153
  def log_data(data)
134
154
  return unless config.log_data
135
155
 
136
- data = utf_encoded(masked(data.dup).to_s) unless data.nil?
137
-
138
156
  if config.prefix_data_lines
139
157
  log('Data:')
140
158
  log_data_lines(data)
@@ -172,22 +190,43 @@ module HttpLog
172
190
  def log_json(data = {})
173
191
  return unless config.json_log
174
192
 
175
- log(json_payload(data).to_json)
193
+ log(
194
+ begin
195
+ dump_json(data)
196
+ rescue
197
+ data[:response_body] = "#{config.json_parser} dump failed"
198
+ data[:request_body] = "#{config.json_parser} dump failed"
199
+ dump_json(data)
200
+ end
201
+ )
202
+ end
203
+
204
+ def dump_json(data)
205
+ config.json_parser.dump(json_payload(data))
176
206
  end
177
207
 
178
- def log_graylog(data = {})
208
+ def log_graylog(data)
179
209
  result = json_payload(data)
180
210
 
181
- result[:rounded_benchmark] = data[:benchmark].round
182
- result[:short_message] = result.delete(:url)
183
- config.logger.public_send(config.logger_method, config.severity, result)
211
+ result[:short_message] = result.delete(:url)
212
+ begin
213
+ send_to_graylog result
214
+ rescue
215
+ result[:response_body] = 'Graylog JSON dump failed'
216
+ result[:request_body] = 'Graylog JSON dump failed'
217
+ send_to_graylog result
218
+ end
219
+ end
220
+
221
+ def send_to_graylog data
222
+ config.logger.public_send(config.logger_method, config.severity, data)
184
223
  end
185
224
 
186
225
  def json_payload(data = {})
187
226
  data[:response_code] = transform_response_code(data[:response_code]) if data[:response_code].is_a?(Symbol)
188
227
 
189
228
  parsed_body = begin
190
- parse_body(data[:response_body], data[:encoding], data[:content_type])
229
+ parse_body(data[:response_body], data[:mask_body], data[:encoding], data[:content_type])
191
230
  rescue BodyParsingError => e
192
231
  e.message
193
232
  end
@@ -203,7 +242,7 @@ module HttpLog
203
242
  {
204
243
  method: data[:method].to_s.upcase,
205
244
  url: masked(data[:url]),
206
- request_body: masked(data[:request_body]),
245
+ request_body: data[:request_body],
207
246
  request_headers: masked(data[:request_headers].to_h),
208
247
  response_code: data[:response_code].to_i,
209
248
  response_body: parsed_body,
@@ -236,6 +275,38 @@ module HttpLog
236
275
  end
237
276
  end
238
277
 
278
+ def parse_request(options)
279
+ return if options[:request_body].nil?
280
+
281
+ # Downcase content-type and content-encoding because ::HTTP returns "Content-Type" and "Content-Encoding"
282
+ headers = options[:request_headers].find_all do |header, _|
283
+ %w[content-type Content-Type content-encoding Content-Encoding].include? header
284
+ end.to_h.each_with_object({}) { |(k, v), h| h[k.downcase] = v }
285
+
286
+ copy = options[:request_body].dup
287
+
288
+ options[:request_body] = if text_based?(headers['content-type']) && options[:mask_body]
289
+ begin
290
+ parse_body(copy, options[:mask_body], headers['content-encoding'], headers['content-type'])
291
+ rescue BodyParsingError => e
292
+ log(e.message)
293
+ end
294
+ else
295
+ masked(copy).to_s
296
+ end
297
+ end
298
+
299
+ def masked_data msg
300
+ case msg
301
+ when Hash
302
+ Hash[msg.map { |k, v| [k, config.filter_parameters.include?(k.downcase) ? PARAM_MASK : masked_data(v)] }]
303
+ when Array
304
+ msg.map { |element| masked_data(element) }
305
+ else
306
+ msg
307
+ end
308
+ end
309
+
239
310
  def string_classes
240
311
  @string_classes ||= begin
241
312
  string_classes = [String]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HttpLog
4
- VERSION = '1.3.3'.freeze
4
+ VERSION = '1.4.2'.freeze
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httplog
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.3
4
+ version: 1.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thilo Rusche
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-14 00:00:00.000000000 Z
11
+ date: 2020-02-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ethon
@@ -192,6 +192,20 @@ dependencies:
192
192
  - - "~>"
193
193
  - !ruby/object:Gem::Version
194
194
  version: '1.7'
195
+ - !ruby/object:Gem::Dependency
196
+ name: oj
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: 3.9.2
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: 3.9.2
195
209
  - !ruby/object:Gem::Dependency
196
210
  name: rack
197
211
  requirement: !ruby/object:Gem::Requirement
@@ -260,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
260
274
  - !ruby/object:Gem::Version
261
275
  version: '0'
262
276
  requirements: []
263
- rubygems_version: 3.0.3
277
+ rubygems_version: 3.1.2
264
278
  signing_key:
265
279
  specification_version: 4
266
280
  summary: Log outgoing HTTP requests.