httparty 0.13.7 → 0.20.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of httparty might be problematic. Click here for more details.

Files changed (101) hide show
  1. checksums.yaml +5 -5
  2. data/.editorconfig +18 -0
  3. data/.github/workflows/ci.yml +23 -0
  4. data/.gitignore +2 -0
  5. data/.rubocop_todo.yml +1 -1
  6. data/{History → Changelog.md} +220 -59
  7. data/Gemfile +8 -3
  8. data/README.md +8 -7
  9. data/bin/httparty +3 -1
  10. data/docs/README.md +171 -0
  11. data/examples/README.md +34 -12
  12. data/examples/aaws.rb +7 -3
  13. data/examples/body_stream.rb +14 -0
  14. data/examples/crack.rb +1 -1
  15. data/examples/custom_parsers.rb +5 -1
  16. data/examples/delicious.rb +4 -4
  17. data/examples/headers_and_user_agents.rb +7 -3
  18. data/examples/idn.rb +10 -0
  19. data/examples/logging.rb +4 -4
  20. data/examples/microsoft_graph.rb +52 -0
  21. data/examples/multipart.rb +22 -0
  22. data/examples/peer_cert.rb +9 -0
  23. data/examples/stackexchange.rb +1 -1
  24. data/examples/stream_download.rb +26 -0
  25. data/examples/tripit_sign_in.rb +17 -6
  26. data/examples/twitter.rb +2 -2
  27. data/examples/whoismyrep.rb +1 -1
  28. data/httparty.gemspec +7 -5
  29. data/lib/httparty/connection_adapter.rb +86 -20
  30. data/lib/httparty/cookie_hash.rb +10 -8
  31. data/lib/httparty/decompressor.rb +92 -0
  32. data/lib/httparty/exceptions.rb +8 -2
  33. data/lib/httparty/hash_conversions.rb +30 -8
  34. data/lib/httparty/headers_processor.rb +32 -0
  35. data/lib/httparty/logger/apache_formatter.rb +31 -6
  36. data/lib/httparty/logger/curl_formatter.rb +68 -23
  37. data/lib/httparty/logger/logger.rb +5 -1
  38. data/lib/httparty/logger/logstash_formatter.rb +61 -0
  39. data/lib/httparty/module_inheritable_attributes.rb +6 -4
  40. data/lib/httparty/net_digest_auth.rb +23 -21
  41. data/lib/httparty/parser.rb +25 -14
  42. data/lib/httparty/request/body.rb +98 -0
  43. data/lib/httparty/request/multipart_boundary.rb +13 -0
  44. data/lib/httparty/request.rb +156 -106
  45. data/lib/httparty/response/headers.rb +23 -19
  46. data/lib/httparty/response.rb +92 -13
  47. data/lib/httparty/response_fragment.rb +21 -0
  48. data/lib/httparty/text_encoder.rb +72 -0
  49. data/lib/httparty/utils.rb +13 -0
  50. data/lib/httparty/version.rb +3 -1
  51. data/lib/httparty.rb +98 -34
  52. data/website/css/common.css +1 -1
  53. metadata +34 -113
  54. data/.travis.yml +0 -7
  55. data/features/basic_authentication.feature +0 -20
  56. data/features/command_line.feature +0 -95
  57. data/features/deals_with_http_error_codes.feature +0 -26
  58. data/features/digest_authentication.feature +0 -30
  59. data/features/handles_compressed_responses.feature +0 -27
  60. data/features/handles_multiple_formats.feature +0 -57
  61. data/features/steps/env.rb +0 -27
  62. data/features/steps/httparty_response_steps.rb +0 -52
  63. data/features/steps/httparty_steps.rb +0 -43
  64. data/features/steps/mongrel_helper.rb +0 -127
  65. data/features/steps/remote_service_steps.rb +0 -90
  66. data/features/supports_read_timeout_option.feature +0 -13
  67. data/features/supports_redirection.feature +0 -22
  68. data/features/supports_timeout_option.feature +0 -13
  69. data/spec/fixtures/delicious.xml +0 -23
  70. data/spec/fixtures/empty.xml +0 -0
  71. data/spec/fixtures/google.html +0 -3
  72. data/spec/fixtures/ssl/generate.sh +0 -29
  73. data/spec/fixtures/ssl/generated/1fe462c2.0 +0 -16
  74. data/spec/fixtures/ssl/generated/bogushost.crt +0 -13
  75. data/spec/fixtures/ssl/generated/ca.crt +0 -16
  76. data/spec/fixtures/ssl/generated/ca.key +0 -15
  77. data/spec/fixtures/ssl/generated/selfsigned.crt +0 -14
  78. data/spec/fixtures/ssl/generated/server.crt +0 -13
  79. data/spec/fixtures/ssl/generated/server.key +0 -15
  80. data/spec/fixtures/ssl/openssl-exts.cnf +0 -9
  81. data/spec/fixtures/twitter.csv +0 -2
  82. data/spec/fixtures/twitter.json +0 -1
  83. data/spec/fixtures/twitter.xml +0 -403
  84. data/spec/fixtures/undefined_method_add_node_for_nil.xml +0 -2
  85. data/spec/httparty/connection_adapter_spec.rb +0 -468
  86. data/spec/httparty/cookie_hash_spec.rb +0 -83
  87. data/spec/httparty/exception_spec.rb +0 -38
  88. data/spec/httparty/hash_conversions_spec.rb +0 -41
  89. data/spec/httparty/logger/apache_formatter_spec.rb +0 -41
  90. data/spec/httparty/logger/curl_formatter_spec.rb +0 -18
  91. data/spec/httparty/logger/logger_spec.rb +0 -38
  92. data/spec/httparty/net_digest_auth_spec.rb +0 -230
  93. data/spec/httparty/parser_spec.rb +0 -173
  94. data/spec/httparty/request_spec.rb +0 -1073
  95. data/spec/httparty/response_spec.rb +0 -241
  96. data/spec/httparty/ssl_spec.rb +0 -74
  97. data/spec/httparty_spec.rb +0 -850
  98. data/spec/spec_helper.rb +0 -59
  99. data/spec/support/ssl_test_helper.rb +0 -47
  100. data/spec/support/ssl_test_server.rb +0 -80
  101. data/spec/support/stub_response.rb +0 -49
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTParty
4
+ module Logger
5
+ class LogstashFormatter #:nodoc:
6
+ TAG_NAME = HTTParty.name
7
+
8
+ attr_accessor :level, :logger
9
+
10
+ def initialize(logger, level)
11
+ @logger = logger
12
+ @level = level.to_sym
13
+ end
14
+
15
+ def format(request, response)
16
+ @request = request
17
+ @response = response
18
+
19
+ logger.public_send level, logstash_message
20
+ end
21
+
22
+ private
23
+
24
+ attr_reader :request, :response
25
+
26
+ def logstash_message
27
+ {
28
+ '@timestamp' => current_time,
29
+ '@version' => 1,
30
+ 'content_length' => content_length || '-',
31
+ 'http_method' => http_method,
32
+ 'message' => message,
33
+ 'path' => path,
34
+ 'response_code' => response.code,
35
+ 'severity' => level,
36
+ 'tags' => [TAG_NAME],
37
+ }.to_json
38
+ end
39
+
40
+ def message
41
+ "[#{TAG_NAME}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
42
+ end
43
+
44
+ def current_time
45
+ Time.now.strftime('%Y-%m-%d %H:%M:%S %z')
46
+ end
47
+
48
+ def http_method
49
+ @http_method ||= request.http_method.name.split('::').last.upcase
50
+ end
51
+
52
+ def path
53
+ @path ||= request.path.to_s
54
+ end
55
+
56
+ def content_length
57
+ @content_length ||= response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
58
+ end
59
+ end
60
+ end
61
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module HTTParty
2
4
  module ModuleInheritableAttributes #:nodoc:
3
5
  def self.included(base)
@@ -9,12 +11,12 @@ module HTTParty
9
11
  duplicate = hash.dup
10
12
 
11
13
  duplicate.each_pair do |key, value|
12
- duplicate[key] = if value.is_a?(Hash)
13
- hash_deep_dup(value)
14
+ if value.is_a?(Hash)
15
+ duplicate[key] = hash_deep_dup(value)
14
16
  elsif value.is_a?(Proc)
15
17
  duplicate[key] = value.dup
16
18
  else
17
- value
19
+ duplicate[key] = value
18
20
  end
19
21
  end
20
22
 
@@ -36,7 +38,7 @@ module HTTParty
36
38
  def inherited(subclass)
37
39
  super
38
40
  @mattr_inheritable_attrs.each do |inheritable_attribute|
39
- ivar = "@#{inheritable_attribute}"
41
+ ivar = :"@#{inheritable_attribute}"
40
42
  subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
41
43
 
42
44
  if instance_variable_get(ivar).respond_to?(:merge)
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'digest/md5'
2
4
  require 'net/http'
3
5
 
@@ -12,13 +14,13 @@ module Net
12
14
  response
13
15
  )
14
16
 
15
- @header['Authorization'] = authenticator.authorization_header
16
- @header['cookie'] = append_cookies(authenticator) if response['Set-Cookie']
17
- end
17
+ authenticator.authorization_header.each do |v|
18
+ add_field('Authorization', v)
19
+ end
18
20
 
19
- def append_cookies(authenticator)
20
- cookies = @header['cookie'] ? @header['cookie'] : []
21
- cookies.concat(authenticator.cookie_header)
21
+ authenticator.cookie_header.each do |v|
22
+ add_field('Cookie', v)
23
+ end
22
24
  end
23
25
 
24
26
  class DigestAuthenticator
@@ -44,12 +46,9 @@ module Net
44
46
  header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
45
47
 
46
48
  if qop_present?
47
- fields = [
48
- %(cnonce="#{@cnonce}"),
49
- %(qop="#{@response['qop']}"),
50
- "nc=00000001"
51
- ]
52
- fields.each { |field| header << field }
49
+ header << %(cnonce="#{@cnonce}")
50
+ header << %(qop="#{@response['qop']}")
51
+ header << 'nc=00000001'
53
52
  end
54
53
 
55
54
  header << %(opaque="#{@response['opaque']}") if opaque_present?
@@ -64,12 +63,15 @@ module Net
64
63
 
65
64
  def parse(response_header)
66
65
  header = response_header['www-authenticate']
67
- .gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
66
+
67
+ header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
68
68
 
69
69
  header =~ /Digest (.*)/
70
70
  params = {}
71
- non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
72
- non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
71
+ if $1
72
+ non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
73
+ non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
74
+ end
73
75
  params
74
76
  end
75
77
 
@@ -95,13 +97,13 @@ module Net
95
97
  end
96
98
 
97
99
  def random
98
- format "%x", (Time.now.to_i + rand(65535))
100
+ format '%x', (Time.now.to_i + rand(65535))
99
101
  end
100
102
 
101
103
  def request_digest
102
104
  a = [md5(a1), @response['nonce'], md5(a2)]
103
- a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
104
- md5(a.join(":"))
105
+ a.insert(2, '00000001', @cnonce, @response['qop']) if qop_present?
106
+ md5(a.join(':'))
105
107
  end
106
108
 
107
109
  def md5(str)
@@ -111,11 +113,11 @@ module Net
111
113
  def algorithm_present?
112
114
  @response.key?('algorithm') && !@response['algorithm'].empty?
113
115
  end
114
-
116
+
115
117
  def use_md5_sess?
116
118
  algorithm_present? && @response['algorithm'] == 'MD5-sess'
117
119
  end
118
-
120
+
119
121
  def a1
120
122
  a1_user_realm_pwd = [@username, @response['realm'], @password].join(':')
121
123
  if use_md5_sess?
@@ -126,7 +128,7 @@ module Net
126
128
  end
127
129
 
128
130
  def a2
129
- [@method, @path].join(":")
131
+ [@method, @path].join(':')
130
132
  end
131
133
  end
132
134
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module HTTParty
2
4
  # The default parser used by HTTParty, supports xml, json, html, csv and
3
5
  # plain text.
@@ -38,16 +40,18 @@ module HTTParty
38
40
  # @abstract Read the Custom Parsers section for more information.
39
41
  class Parser
40
42
  SupportedFormats = {
41
- 'text/xml' => :xml,
42
- 'application/xml' => :xml,
43
- 'application/json' => :json,
44
- 'text/json' => :json,
45
- 'application/javascript' => :plain,
46
- 'text/javascript' => :plain,
47
- 'text/html' => :html,
48
- 'text/plain' => :plain,
49
- 'text/csv' => :csv,
50
- 'application/csv' => :csv,
43
+ 'text/xml' => :xml,
44
+ 'application/xml' => :xml,
45
+ 'application/json' => :json,
46
+ 'application/vnd.api+json' => :json,
47
+ 'application/hal+json' => :json,
48
+ 'text/json' => :json,
49
+ 'application/javascript' => :plain,
50
+ 'text/javascript' => :plain,
51
+ 'text/html' => :html,
52
+ 'text/plain' => :plain,
53
+ 'text/csv' => :csv,
54
+ 'application/csv' => :csv,
51
55
  'text/comma-separated-values' => :csv
52
56
  }
53
57
 
@@ -99,8 +103,11 @@ module HTTParty
99
103
  # @return [nil] when the response body is nil, an empty string, spaces only or "null"
100
104
  def parse
101
105
  return nil if body.nil?
102
- return nil if body == "null"
106
+ return nil if body == 'null'
103
107
  return nil if body.valid_encoding? && body.strip.empty?
108
+ if body.valid_encoding? && body.encoding == Encoding::UTF_8
109
+ @body = body.gsub(/\A#{UTF8_BOM}/, '')
110
+ end
104
111
  if supports_format?
105
112
  parse_supported_format
106
113
  else
@@ -114,6 +121,8 @@ module HTTParty
114
121
  MultiXml.parse(body)
115
122
  end
116
123
 
124
+ UTF8_BOM = "\xEF\xBB\xBF"
125
+
117
126
  def json
118
127
  JSON.parse(body, :quirks_mode => true, :allow_nan => true)
119
128
  end
@@ -135,9 +144,11 @@ module HTTParty
135
144
  end
136
145
 
137
146
  def parse_supported_format
138
- send(format)
139
- rescue NoMethodError => e
140
- raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
147
+ if respond_to?(format, true)
148
+ send(format)
149
+ else
150
+ raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
151
+ end
141
152
  end
142
153
  end
143
154
  end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'multipart_boundary'
4
+
5
+ module HTTParty
6
+ class Request
7
+ class Body
8
+ NEWLINE = "\r\n"
9
+ private_constant :NEWLINE
10
+
11
+ def initialize(params, query_string_normalizer: nil, force_multipart: false)
12
+ @params = params
13
+ @query_string_normalizer = query_string_normalizer
14
+ @force_multipart = force_multipart
15
+ end
16
+
17
+ def call
18
+ if params.respond_to?(:to_hash)
19
+ multipart? ? generate_multipart : normalize_query(params)
20
+ else
21
+ params
22
+ end
23
+ end
24
+
25
+ def boundary
26
+ @boundary ||= MultipartBoundary.generate
27
+ end
28
+
29
+ def multipart?
30
+ params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
31
+ end
32
+
33
+ private
34
+
35
+ def generate_multipart
36
+ normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
37
+
38
+ multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
39
+ memo << "--#{boundary}#{NEWLINE}"
40
+ memo << %(Content-Disposition: form-data; name="#{key}")
41
+ # value.path is used to support ActionDispatch::Http::UploadedFile
42
+ # https://github.com/jnunemaker/httparty/pull/585
43
+ memo << %(; filename="#{file_name(value)}") if file?(value)
44
+ memo << NEWLINE
45
+ memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
46
+ memo << NEWLINE
47
+ memo << content_body(value)
48
+ memo << NEWLINE
49
+ end
50
+
51
+ multipart << "--#{boundary}--#{NEWLINE}"
52
+ end
53
+
54
+ def has_file?(value)
55
+ if value.respond_to?(:to_hash)
56
+ value.to_hash.any? { |_, v| has_file?(v) }
57
+ elsif value.respond_to?(:to_ary)
58
+ value.to_ary.any? { |v| has_file?(v) }
59
+ else
60
+ file?(value)
61
+ end
62
+ end
63
+
64
+ def file?(object)
65
+ object.respond_to?(:path) && object.respond_to?(:read)
66
+ end
67
+
68
+ def normalize_query(query)
69
+ if query_string_normalizer
70
+ query_string_normalizer.call(query)
71
+ else
72
+ HashConversions.to_params(query)
73
+ end
74
+ end
75
+
76
+ def content_body(object)
77
+ if file?(object)
78
+ object = (file = object).read
79
+ file.rewind if file.respond_to?(:rewind)
80
+ end
81
+
82
+ object.to_s
83
+ end
84
+
85
+ def content_type(object)
86
+ return object.content_type if object.respond_to?(:content_type)
87
+ mime = MIME::Types.type_for(object.path)
88
+ mime.empty? ? 'application/octet-stream' : mime[0].content_type
89
+ end
90
+
91
+ def file_name(object)
92
+ object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
93
+ end
94
+
95
+ attr_reader :params, :query_string_normalizer, :force_multipart
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'securerandom'
4
+
5
+ module HTTParty
6
+ class Request
7
+ class MultipartBoundary
8
+ def self.generate
9
+ "------------------------#{SecureRandom.urlsafe_base64(12)}"
10
+ end
11
+ end
12
+ end
13
+ end