yoyle439587298 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +10 -0
  3. data/.travis.yml +7 -0
  4. data/Gemfile +14 -0
  5. data/Guardfile +16 -0
  6. data/History +303 -0
  7. data/LICENSE +21 -0
  8. data/MIT-LICENSE +20 -0
  9. data/README.md +80 -0
  10. data/Rakefile +12 -0
  11. data/bin/httparty +117 -0
  12. data/cucumber.yml +1 -0
  13. data/examples/aaws.rb +32 -0
  14. data/examples/basic.rb +32 -0
  15. data/examples/crack.rb +19 -0
  16. data/examples/custom_parsers.rb +67 -0
  17. data/examples/delicious.rb +37 -0
  18. data/examples/google.rb +16 -0
  19. data/examples/headers_and_user_agents.rb +6 -0
  20. data/examples/nokogiri_html_parser.rb +22 -0
  21. data/examples/rubyurl.rb +14 -0
  22. data/examples/tripit_sign_in.rb +33 -0
  23. data/examples/twitter.rb +31 -0
  24. data/examples/whoismyrep.rb +10 -0
  25. data/features/basic_authentication.feature +20 -0
  26. data/features/command_line.feature +7 -0
  27. data/features/deals_with_http_error_codes.feature +26 -0
  28. data/features/digest_authentication.feature +20 -0
  29. data/features/handles_compressed_responses.feature +27 -0
  30. data/features/handles_multiple_formats.feature +57 -0
  31. data/features/steps/env.rb +22 -0
  32. data/features/steps/httparty_response_steps.rb +52 -0
  33. data/features/steps/httparty_steps.rb +35 -0
  34. data/features/steps/mongrel_helper.rb +94 -0
  35. data/features/steps/remote_service_steps.rb +74 -0
  36. data/features/supports_redirection.feature +22 -0
  37. data/features/supports_timeout_option.feature +13 -0
  38. data/httparty.gemspec +26 -0
  39. data/lib/httparty.rb +578 -0
  40. data/lib/httparty/connection_adapter.rb +176 -0
  41. data/lib/httparty/cookie_hash.rb +22 -0
  42. data/lib/httparty/core_extensions.rb +32 -0
  43. data/lib/httparty/exceptions.rb +29 -0
  44. data/lib/httparty/hash_conversions.rb +51 -0
  45. data/lib/httparty/logger/apache_logger.rb +22 -0
  46. data/lib/httparty/logger/curl_logger.rb +48 -0
  47. data/lib/httparty/logger/logger.rb +18 -0
  48. data/lib/httparty/module_inheritable_attributes.rb +56 -0
  49. data/lib/httparty/net_digest_auth.rb +84 -0
  50. data/lib/httparty/parser.rb +141 -0
  51. data/lib/httparty/request.rb +330 -0
  52. data/lib/httparty/response.rb +72 -0
  53. data/lib/httparty/response/headers.rb +31 -0
  54. data/lib/httparty/version.rb +3 -0
  55. data/script/release +42 -0
  56. data/spec/fixtures/delicious.xml +23 -0
  57. data/spec/fixtures/empty.xml +0 -0
  58. data/spec/fixtures/google.html +3 -0
  59. data/spec/fixtures/ssl/generate.sh +29 -0
  60. data/spec/fixtures/ssl/generated/1fe462c2.0 +16 -0
  61. data/spec/fixtures/ssl/generated/bogushost.crt +13 -0
  62. data/spec/fixtures/ssl/generated/ca.crt +16 -0
  63. data/spec/fixtures/ssl/generated/ca.key +15 -0
  64. data/spec/fixtures/ssl/generated/selfsigned.crt +14 -0
  65. data/spec/fixtures/ssl/generated/server.crt +13 -0
  66. data/spec/fixtures/ssl/generated/server.key +15 -0
  67. data/spec/fixtures/ssl/openssl-exts.cnf +9 -0
  68. data/spec/fixtures/twitter.csv +2 -0
  69. data/spec/fixtures/twitter.json +1 -0
  70. data/spec/fixtures/twitter.xml +403 -0
  71. data/spec/fixtures/undefined_method_add_node_for_nil.xml +2 -0
  72. data/spec/httparty/connection_adapter_spec.rb +298 -0
  73. data/spec/httparty/cookie_hash_spec.rb +83 -0
  74. data/spec/httparty/exception_spec.rb +23 -0
  75. data/spec/httparty/logger/apache_logger_spec.rb +26 -0
  76. data/spec/httparty/logger/curl_logger_spec.rb +18 -0
  77. data/spec/httparty/logger/logger_spec.rb +22 -0
  78. data/spec/httparty/net_digest_auth_spec.rb +152 -0
  79. data/spec/httparty/parser_spec.rb +165 -0
  80. data/spec/httparty/request_spec.rb +631 -0
  81. data/spec/httparty/response_spec.rb +221 -0
  82. data/spec/httparty/ssl_spec.rb +74 -0
  83. data/spec/httparty_spec.rb +764 -0
  84. data/spec/spec.opts +2 -0
  85. data/spec/spec_helper.rb +37 -0
  86. data/spec/support/ssl_test_helper.rb +47 -0
  87. data/spec/support/ssl_test_server.rb +80 -0
  88. data/spec/support/stub_response.rb +43 -0
  89. data/website/css/common.css +47 -0
  90. data/website/index.html +73 -0
  91. metadata +208 -0
@@ -0,0 +1,84 @@
1
+ require 'digest/md5'
2
+ require 'net/http'
3
+
4
+ module Net
5
+ module HTTPHeader
6
+ def digest_auth(username, password, response)
7
+ @header['Authorization'] = DigestAuthenticator.new(username, password,
8
+ @method, @path, response).authorization_header
9
+ end
10
+
11
+
12
+ class DigestAuthenticator
13
+ def initialize(username, password, method, path, response_header)
14
+ @username = username
15
+ @password = password
16
+ @method = method
17
+ @path = path
18
+ @response = parse(response_header)
19
+ end
20
+
21
+ def authorization_header
22
+ @cnonce = md5(random)
23
+ header = [
24
+ %Q(Digest username="#{@username}"),
25
+ %Q(realm="#{@response['realm']}"),
26
+ %Q(nonce="#{@response['nonce']}"),
27
+ %Q(uri="#{@path}"),
28
+ %Q(response="#{request_digest}"),
29
+ ]
30
+
31
+ if qop_present?
32
+ fields = [
33
+ %Q(cnonce="#{@cnonce}"),
34
+ %Q(qop="#{@response['qop']}"),
35
+ %Q(nc=00000001)
36
+ ]
37
+ fields.each { |field| header << field }
38
+ end
39
+
40
+ header << %Q(opaque="#{@response['opaque']}") if opaque_present?
41
+ header
42
+ end
43
+
44
+ private
45
+
46
+ def parse(response_header)
47
+ response_header['www-authenticate'] =~ /Digest (.*)/
48
+ params = {}
49
+ $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
50
+ params
51
+ end
52
+
53
+ def opaque_present?
54
+ @response.has_key?('opaque') and not @response['opaque'].empty?
55
+ end
56
+
57
+ def qop_present?
58
+ @response.has_key?('qop') and not @response['qop'].empty?
59
+ end
60
+
61
+ def random
62
+ "%x" % (Time.now.to_i + rand(65535))
63
+ end
64
+
65
+ def request_digest
66
+ a = [md5(a1), @response['nonce'], md5(a2)]
67
+ a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
68
+ md5(a.join(":"))
69
+ end
70
+
71
+ def md5(str)
72
+ Digest::MD5.hexdigest(str)
73
+ end
74
+
75
+ def a1
76
+ [@username, @response['realm'], @password].join(":")
77
+ end
78
+
79
+ def a2
80
+ [@method, @path].join(":")
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,141 @@
1
+ module HTTParty
2
+ # The default parser used by HTTParty, supports xml, json, html, csv and
3
+ # plain text.
4
+ #
5
+ # == Custom Parsers
6
+ #
7
+ # If you'd like to do your own custom parsing, subclassing HTTParty::Parser
8
+ # will make that process much easier. There are a few different ways you can
9
+ # utilize HTTParty::Parser as a superclass.
10
+ #
11
+ # @example Intercept the parsing for all formats
12
+ # class SimpleParser < HTTParty::Parser
13
+ # def parse
14
+ # perform_parsing
15
+ # end
16
+ # end
17
+ #
18
+ # @example Add the atom format and parsing method to the default parser
19
+ # class AtomParsingIncluded < HTTParty::Parser
20
+ # SupportedFormats.merge!(
21
+ # {"application/atom+xml" => :atom}
22
+ # )
23
+ #
24
+ # def atom
25
+ # perform_atom_parsing
26
+ # end
27
+ # end
28
+ #
29
+ # @example Only support the atom format
30
+ # class ParseOnlyAtom < HTTParty::Parser
31
+ # SupportedFormats = {"application/atom+xml" => :atom}
32
+ #
33
+ # def atom
34
+ # perform_atom_parsing
35
+ # end
36
+ # end
37
+ #
38
+ # @abstract Read the Custom Parsers section for more information.
39
+ class Parser
40
+ SupportedFormats = {
41
+ 'text/xml' => :xml,
42
+ 'application/xml' => :xml,
43
+ 'application/json' => :json,
44
+ 'text/json' => :json,
45
+ 'application/javascript' => :plain,
46
+ 'text/javascript' => :plain,
47
+ 'text/html' => :html,
48
+ 'text/plain' => :plain,
49
+ 'text/csv' => :csv,
50
+ 'application/csv' => :csv,
51
+ 'text/comma-separated-values' => :csv
52
+ }
53
+
54
+ # The response body of the request
55
+ # @return [String]
56
+ attr_reader :body
57
+
58
+ # The intended parsing format for the request
59
+ # @return [Symbol] e.g. :json
60
+ attr_reader :format
61
+
62
+ # Instantiate the parser and call {#parse}.
63
+ # @param [String] body the response body
64
+ # @param [Symbol] format the response format
65
+ # @return parsed response
66
+ def self.call(body, format)
67
+ new(body, format).parse
68
+ end
69
+
70
+ # @return [Hash] the SupportedFormats hash
71
+ def self.formats
72
+ const_get(:SupportedFormats)
73
+ end
74
+
75
+ # @param [String] mimetype response MIME type
76
+ # @return [Symbol]
77
+ # @return [nil] mime type not supported
78
+ def self.format_from_mimetype(mimetype)
79
+ formats[formats.keys.detect {|k| mimetype.include?(k)}]
80
+ end
81
+
82
+ # @return [Array<Symbol>] list of supported formats
83
+ def self.supported_formats
84
+ formats.values.uniq
85
+ end
86
+
87
+ # @param [Symbol] format e.g. :json, :xml
88
+ # @return [Boolean]
89
+ def self.supports_format?(format)
90
+ supported_formats.include?(format)
91
+ end
92
+
93
+ def initialize(body, format)
94
+ @body = body
95
+ @format = format
96
+ end
97
+
98
+ # @return [Object] the parsed body
99
+ # @return [nil] when the response body is nil, an empty string, spaces only or "null"
100
+ def parse
101
+ return nil if body.nil? || body.strip.empty? || body == "null"
102
+ if supports_format?
103
+ parse_supported_format
104
+ else
105
+ body
106
+ end
107
+ end
108
+
109
+ protected
110
+
111
+ def xml
112
+ MultiXml.parse(body)
113
+ end
114
+
115
+ def json
116
+ JSON.load(body, nil)
117
+ end
118
+
119
+ def csv
120
+ CSV.parse(body)
121
+ end
122
+
123
+ def html
124
+ body
125
+ end
126
+
127
+ def plain
128
+ body
129
+ end
130
+
131
+ def supports_format?
132
+ self.class.supports_format?(format)
133
+ end
134
+
135
+ def parse_supported_format
136
+ send(format)
137
+ rescue NoMethodError => e
138
+ raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,330 @@
1
+ module HTTParty
2
+ class Request #:nodoc:
3
+ SupportedHTTPMethods = [
4
+ Net::HTTP::Get,
5
+ Net::HTTP::Post,
6
+ Net::HTTP::Patch,
7
+ Net::HTTP::Put,
8
+ Net::HTTP::Delete,
9
+ Net::HTTP::Head,
10
+ Net::HTTP::Options,
11
+ Net::HTTP::Move,
12
+ Net::HTTP::Copy
13
+ ]
14
+
15
+ SupportedURISchemes = [URI::HTTP, URI::HTTPS, URI::Generic]
16
+
17
+ NON_RAILS_QUERY_STRING_NORMALIZER = Proc.new do |query|
18
+ Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
19
+ if value.nil?
20
+ key.to_s
21
+ elsif value.is_a?(Array)
22
+ value.map {|v| "#{key}=#{URI.encode(v.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))}"}
23
+ else
24
+ HashConversions.to_params(key => value)
25
+ end
26
+ end.flatten.join('&')
27
+ end
28
+
29
+ attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
30
+ attr_reader :path
31
+
32
+ def initialize(http_method, path, o={})
33
+ self.http_method = http_method
34
+ self.path = path
35
+ self.options = {
36
+ :limit => o.delete(:no_follow) ? 1 : 5,
37
+ :assume_utf16_is_big_endian => true,
38
+ :default_params => {},
39
+ :follow_redirects => true,
40
+ :parser => Parser,
41
+ :connection_adapter => ConnectionAdapter
42
+ }.merge(o)
43
+ end
44
+
45
+ def path=(uri)
46
+ @path = URI.parse(uri)
47
+ end
48
+
49
+ def request_uri(uri)
50
+ if uri.respond_to? :request_uri
51
+ uri.request_uri
52
+ else
53
+ uri.path
54
+ end
55
+ end
56
+
57
+ def uri
58
+ new_uri = path.relative? ? URI.parse("#{base_uri}#{path}") : path.clone
59
+
60
+ # avoid double query string on redirects [#12]
61
+ unless redirect
62
+ new_uri.query = query_string(new_uri)
63
+ end
64
+
65
+ unless SupportedURISchemes.include? new_uri.class
66
+ raise UnsupportedURIScheme, "'#{new_uri}' Must be HTTP, HTTPS or Generic"
67
+ end
68
+
69
+ @last_uri = new_uri
70
+ end
71
+
72
+ def base_uri
73
+ redirect ? "#{@last_uri.scheme}://#{@last_uri.host}" : options[:base_uri]
74
+ end
75
+
76
+ def format
77
+ options[:format] || (format_from_mimetype(last_response['content-type']) if last_response)
78
+ end
79
+
80
+ def parser
81
+ options[:parser]
82
+ end
83
+
84
+ def connection_adapter
85
+ options[:connection_adapter]
86
+ end
87
+
88
+ def perform(&block)
89
+ validate
90
+ setup_raw_request
91
+ chunked_body = nil
92
+
93
+ self.last_response = http.request(@raw_request) do |http_response|
94
+ if block
95
+ chunks = []
96
+
97
+ http_response.read_body do |fragment|
98
+ chunks << fragment
99
+ block.call(fragment)
100
+ end
101
+
102
+ chunked_body = chunks.join
103
+ end
104
+ end
105
+
106
+ handle_deflation unless http_method == Net::HTTP::Head
107
+ handle_response(chunked_body, &block)
108
+ end
109
+
110
+ def raw_body
111
+ @raw_request.body
112
+ end
113
+
114
+ private
115
+
116
+ def http
117
+ connection_adapter.call(uri, options)
118
+ end
119
+
120
+ def body
121
+ options[:body].is_a?(Hash) ? normalize_query(options[:body]) : options[:body]
122
+ end
123
+
124
+ def credentials
125
+ options[:basic_auth] || options[:digest_auth]
126
+ end
127
+
128
+ def username
129
+ credentials[:username]
130
+ end
131
+
132
+ def password
133
+ credentials[:password]
134
+ end
135
+
136
+ def normalize_query(query)
137
+ if query_string_normalizer
138
+ query_string_normalizer.call(query)
139
+ else
140
+ HashConversions.to_params(query)
141
+ end
142
+ end
143
+
144
+ def query_string_normalizer
145
+ options[:query_string_normalizer]
146
+ end
147
+
148
+ def setup_raw_request
149
+ @raw_request = http_method.new(request_uri(uri))
150
+ @raw_request.body = body if body
151
+ @raw_request.initialize_http_header(options[:headers])
152
+ @raw_request.basic_auth(username, password) if options[:basic_auth]
153
+ setup_digest_auth if options[:digest_auth]
154
+ end
155
+
156
+ def setup_digest_auth
157
+ auth_request = http_method.new(uri.request_uri)
158
+ auth_request.initialize_http_header(options[:headers])
159
+ res = http.request(auth_request)
160
+
161
+ if res['www-authenticate'] != nil && res['www-authenticate'].length > 0
162
+ @raw_request.digest_auth(username, password, res)
163
+ end
164
+ end
165
+
166
+ def query_string(uri)
167
+ query_string_parts = []
168
+ query_string_parts << uri.query unless uri.query.nil?
169
+
170
+ if options[:query].is_a?(Hash)
171
+ query_string_parts << normalize_query(options[:default_params].merge(options[:query]))
172
+ else
173
+ query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
174
+ query_string_parts << options[:query] unless options[:query].nil?
175
+ end
176
+
177
+ query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
178
+ query_string_parts.size > 0 ? query_string_parts.join('&') : nil
179
+ end
180
+
181
+ def get_charset
182
+ content_type = last_response["content-type"]
183
+ if content_type.nil?
184
+ return nil
185
+ end
186
+
187
+ if content_type =~ /;\s*charset\s*=\s*([^=,;"\s]+)/i
188
+ return $1
189
+ end
190
+
191
+ if content_type =~ /;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i
192
+ return $1.gsub(/\\(.)/, '\1')
193
+ end
194
+
195
+ nil
196
+ end
197
+
198
+ def encode_with_ruby_encoding(body, charset)
199
+ begin
200
+ encoding = Encoding.find(charset)
201
+ body.force_encoding(encoding)
202
+ rescue
203
+ body
204
+ end
205
+ end
206
+
207
+ def assume_utf16_is_big_endian
208
+ options[:assume_utf16_is_big_endian]
209
+ end
210
+
211
+ def encode_utf_16(body)
212
+ if body.bytesize >= 2
213
+ if body.getbyte(0) == 0xFF && body.getbyte(1) == 0xFE
214
+ return body.force_encoding("UTF-16LE")
215
+ elsif body.getbyte(0) == 0xFE && body.getbyte(1) == 0xFF
216
+ return body.force_encoding("UTF-16BE")
217
+ end
218
+ end
219
+
220
+ if assume_utf16_is_big_endian
221
+ body.force_encoding("UTF-16BE")
222
+ else
223
+ body.force_encoding("UTF-16LE")
224
+ end
225
+
226
+ end
227
+
228
+ def _encode_body(body)
229
+ charset = get_charset
230
+
231
+ if charset.nil?
232
+ return body
233
+ end
234
+
235
+ if "utf-16".casecmp(charset) == 0
236
+ encode_utf_16(body)
237
+ else
238
+ encode_with_ruby_encoding(body, charset)
239
+ end
240
+ end
241
+
242
+ def encode_body(body)
243
+ if "".respond_to?(:encoding)
244
+ _encode_body(body)
245
+ else
246
+ body
247
+ end
248
+ end
249
+
250
+ def handle_response(body, &block)
251
+ if response_redirects?
252
+ options[:limit] -= 1
253
+ if options[:logger]
254
+ logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
255
+ logger.format(self, last_response)
256
+ end
257
+ self.path = last_response['location']
258
+ self.redirect = true
259
+ self.http_method = Net::HTTP::Get unless options[:maintain_method_across_redirects]
260
+ capture_cookies(last_response)
261
+ perform(&block)
262
+ else
263
+ body = body || last_response.body
264
+ body = encode_body(body)
265
+ Response.new(self, last_response, lambda { parse_response(body) }, :body => body)
266
+ end
267
+ end
268
+
269
+ # Inspired by Ruby 1.9
270
+ def handle_deflation
271
+ case last_response["content-encoding"]
272
+ when "gzip", "x-gzip"
273
+ body_io = StringIO.new(last_response.body)
274
+ last_response.body.replace Zlib::GzipReader.new(body_io).read
275
+ last_response.delete('content-encoding')
276
+ when "deflate"
277
+ last_response.body.replace Zlib::Inflate.inflate(last_response.body)
278
+ last_response.delete('content-encoding')
279
+ end
280
+ end
281
+
282
+ def response_redirects?
283
+ case last_response
284
+ when Net::HTTPMultipleChoice, # 300
285
+ Net::HTTPMovedPermanently, # 301
286
+ Net::HTTPFound, # 302
287
+ Net::HTTPSeeOther, # 303
288
+ Net::HTTPUseProxy, # 305
289
+ Net::HTTPTemporaryRedirect
290
+ options[:follow_redirects] && last_response.key?('location')
291
+ end
292
+ end
293
+
294
+ def parse_response(body)
295
+ parser.call(body, format)
296
+ end
297
+
298
+ def capture_cookies(response)
299
+ return unless response['Set-Cookie']
300
+ cookies_hash = HTTParty::CookieHash.new()
301
+ cookies_hash.add_cookies(options[:headers]['Cookie']) if options[:headers] && options[:headers]['Cookie']
302
+ response.get_fields('Set-Cookie').each { |cookie| cookies_hash.add_cookies(cookie) }
303
+ options[:headers] ||= {}
304
+ options[:headers]['Cookie'] = cookies_hash.to_cookie_string
305
+ end
306
+
307
+ # Uses the HTTP Content-Type header to determine the format of the
308
+ # response It compares the MIME type returned to the types stored in the
309
+ # SupportedFormats hash
310
+ def format_from_mimetype(mimetype)
311
+ if mimetype && parser.respond_to?(:format_from_mimetype)
312
+ parser.format_from_mimetype(mimetype)
313
+ end
314
+ end
315
+
316
+ def validate
317
+ raise HTTParty::RedirectionTooDeep.new(last_response), 'HTTP redirects too deep' if options[:limit].to_i <= 0
318
+ raise ArgumentError, 'only get, post, patch, put, delete, head, and options methods are supported' unless SupportedHTTPMethods.include?(http_method)
319
+ raise ArgumentError, ':headers must be a hash' if options[:headers] && !options[:headers].is_a?(Hash)
320
+ raise ArgumentError, 'only one authentication method, :basic_auth or :digest_auth may be used at a time' if options[:basic_auth] && options[:digest_auth]
321
+ raise ArgumentError, ':basic_auth must be a hash' if options[:basic_auth] && !options[:basic_auth].is_a?(Hash)
322
+ raise ArgumentError, ':digest_auth must be a hash' if options[:digest_auth] && !options[:digest_auth].is_a?(Hash)
323
+ raise ArgumentError, ':query must be hash if using HTTP Post' if post? && !options[:query].nil? && !options[:query].is_a?(Hash)
324
+ end
325
+
326
+ def post?
327
+ Net::HTTP::Post == http_method
328
+ end
329
+ end
330
+ end