httpotato 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (70) hide show
  1. data/.gitignore +7 -0
  2. data/History +216 -0
  3. data/MIT-LICENSE +20 -0
  4. data/Manifest +47 -0
  5. data/README.rdoc +52 -0
  6. data/Rakefile +90 -0
  7. data/VERSION +1 -0
  8. data/bin/httpotato +108 -0
  9. data/cucumber.yml +1 -0
  10. data/examples/aaws.rb +32 -0
  11. data/examples/basic.rb +11 -0
  12. data/examples/custom_parsers.rb +67 -0
  13. data/examples/delicious.rb +37 -0
  14. data/examples/google.rb +16 -0
  15. data/examples/rubyurl.rb +14 -0
  16. data/examples/twitter.rb +31 -0
  17. data/examples/whoismyrep.rb +10 -0
  18. data/features/basic_authentication.feature +20 -0
  19. data/features/command_line.feature +7 -0
  20. data/features/deals_with_http_error_codes.feature +26 -0
  21. data/features/digest_authentication.feature +20 -0
  22. data/features/handles_compressed_responses.feature +19 -0
  23. data/features/handles_multiple_formats.feature +34 -0
  24. data/features/steps/env.rb +23 -0
  25. data/features/steps/httpotato_response_steps.rb +26 -0
  26. data/features/steps/httpotato_steps.rb +27 -0
  27. data/features/steps/mongrel_helper.rb +94 -0
  28. data/features/steps/remote_service_steps.rb +69 -0
  29. data/features/supports_redirection.feature +22 -0
  30. data/features/supports_timeout_option.feature +13 -0
  31. data/httpotato.gemspec +150 -0
  32. data/lib/httpotato.rb +371 -0
  33. data/lib/httpotato/cookie_hash.rb +22 -0
  34. data/lib/httpotato/core_extensions.rb +31 -0
  35. data/lib/httpotato/exceptions.rb +26 -0
  36. data/lib/httpotato/module_inheritable_attributes.rb +34 -0
  37. data/lib/httpotato/net_digest_auth.rb +35 -0
  38. data/lib/httpotato/parser.rb +146 -0
  39. data/lib/httpotato/request.rb +231 -0
  40. data/lib/httpotato/response.rb +79 -0
  41. data/spec/fixtures/delicious.xml +23 -0
  42. data/spec/fixtures/empty.xml +0 -0
  43. data/spec/fixtures/google.html +3 -0
  44. data/spec/fixtures/ssl/generate.sh +29 -0
  45. data/spec/fixtures/ssl/generated/1fe462c2.0 +15 -0
  46. data/spec/fixtures/ssl/generated/bogushost.crt +13 -0
  47. data/spec/fixtures/ssl/generated/ca.crt +15 -0
  48. data/spec/fixtures/ssl/generated/ca.key +15 -0
  49. data/spec/fixtures/ssl/generated/selfsigned.crt +14 -0
  50. data/spec/fixtures/ssl/generated/server.crt +13 -0
  51. data/spec/fixtures/ssl/generated/server.key +15 -0
  52. data/spec/fixtures/ssl/openssl-exts.cnf +9 -0
  53. data/spec/fixtures/twitter.json +1 -0
  54. data/spec/fixtures/twitter.xml +403 -0
  55. data/spec/fixtures/undefined_method_add_node_for_nil.xml +2 -0
  56. data/spec/httpotato/cookie_hash_spec.rb +71 -0
  57. data/spec/httpotato/parser_spec.rb +155 -0
  58. data/spec/httpotato/request_spec.rb +430 -0
  59. data/spec/httpotato/response_spec.rb +188 -0
  60. data/spec/httpotato/ssl_spec.rb +54 -0
  61. data/spec/httpotato_spec.rb +570 -0
  62. data/spec/spec.opts +3 -0
  63. data/spec/spec_helper.rb +20 -0
  64. data/spec/support/ssl_test_helper.rb +25 -0
  65. data/spec/support/ssl_test_server.rb +69 -0
  66. data/spec/support/stub_response.rb +30 -0
  67. data/test.rb +39 -0
  68. data/website/css/common.css +47 -0
  69. data/website/index.html +66 -0
  70. metadata +260 -0
@@ -0,0 +1,22 @@
1
+ class HTTPotato::CookieHash < Hash #:nodoc:
2
+
3
+ CLIENT_COOKIES = %w{path expires domain path secure HTTPOnly}
4
+
5
+ def add_cookies(value)
6
+ case value
7
+ when Hash
8
+ merge!(value)
9
+ when String
10
+ value.split('; ').each do |cookie|
11
+ array = cookie.split('=')
12
+ self[array[0].to_sym] = array[1]
13
+ end
14
+ else
15
+ raise "add_cookies only takes a Hash or a String"
16
+ end
17
+ end
18
+
19
+ def to_cookie_string
20
+ delete_if { |k, v| CLIENT_COOKIES.include?(k.to_s) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
21
+ end
22
+ end
@@ -0,0 +1,31 @@
1
+ module HTTPotato
2
+ if defined?(::BasicObject)
3
+ BasicObject = ::BasicObject #:nodoc:
4
+ else
5
+ class BasicObject #:nodoc:
6
+ instance_methods.each { |m| undef_method m unless m =~ /^__|instance_eval/ }
7
+ end
8
+ end
9
+ end
10
+
11
+ # 1.8.6 has mistyping of transitive in if statement
12
+ require "rexml/document"
13
+ module REXML #:nodoc:
14
+ class Document < Element #:nodoc:
15
+ def write( output=$stdout, indent=-1, transitive=false, ie_hack=false )
16
+ if xml_decl.encoding != "UTF-8" && !output.kind_of?(Output)
17
+ output = Output.new( output, xml_decl.encoding )
18
+ end
19
+ formatter = if indent > -1
20
+ if transitive
21
+ REXML::Formatters::Transitive.new( indent, ie_hack )
22
+ else
23
+ REXML::Formatters::Pretty.new( indent, ie_hack )
24
+ end
25
+ else
26
+ REXML::Formatters::Default.new( ie_hack )
27
+ end
28
+ formatter.write( self, output )
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,26 @@
1
+ module HTTPotato
2
+ # Exception raised when you attempt to set a non-existant format
3
+ class UnsupportedFormat < StandardError; end
4
+
5
+ # Exception raised when using a URI scheme other than HTTP or HTTPS
6
+ class UnsupportedURIScheme < StandardError; end
7
+
8
+ # @abstract Exceptions which inherit from ResponseError contain the Net::HTTP
9
+ # response object accessible via the {#response} method.
10
+ class ResponseError < StandardError
11
+ # Returns the response of the last request
12
+ # @return [Net::HTTPResponse] A subclass of Net::HTTPResponse, e.g.
13
+ # Net::HTTPOK
14
+ attr_reader :response
15
+
16
+ # Instantiate an instance of ResponseError with a Net::HTTPResponse object
17
+ # @param [Net::HTTPResponse]
18
+ def initialize(response)
19
+ @response = response
20
+ end
21
+ end
22
+
23
+ # Exception that is raised when request has redirected too many times.
24
+ # Calling {#response} returns the Net:HTTP response object.
25
+ class RedirectionTooDeep < ResponseError; end
26
+ end
@@ -0,0 +1,34 @@
1
+ module HTTPotato
2
+ module ModuleInheritableAttributes #:nodoc:
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ end
6
+
7
+ module ClassMethods #:nodoc:
8
+ def mattr_inheritable(*args)
9
+ @mattr_inheritable_attrs ||= [:mattr_inheritable_attrs]
10
+ @mattr_inheritable_attrs += args
11
+ args.each do |arg|
12
+ module_eval %(class << self; attr_accessor :#{arg} end)
13
+ end
14
+ @mattr_inheritable_attrs
15
+ end
16
+
17
+ def inherited(subclass)
18
+ super
19
+ @mattr_inheritable_attrs.each do |inheritable_attribute|
20
+ ivar = "@#{inheritable_attribute}"
21
+ subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
22
+ if instance_variable_get(ivar).respond_to?(:merge)
23
+ method = <<-EOM
24
+ def self.#{inheritable_attribute}
25
+ #{ivar} = superclass.#{inheritable_attribute}.merge #{ivar}
26
+ end
27
+ EOM
28
+ subclass.class_eval method
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,35 @@
1
+ require 'digest/md5'
2
+ require 'net/http'
3
+
4
+ module Net
5
+ module HTTPHeader
6
+ def digest_auth(user, password, response)
7
+ response['www-authenticate'] =~ /^(\w+) (.*)/
8
+
9
+ params = {}
10
+ $2.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
11
+ params.merge!("cnonce" => Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535))))
12
+
13
+ a_1 = Digest::MD5.hexdigest("#{user}:#{params['realm']}:#{password}")
14
+ a_2 = Digest::MD5.hexdigest("#{@method}:#{@path}")
15
+
16
+ request_digest = Digest::MD5.hexdigest(
17
+ [a_1, params['nonce'], "0", params['cnonce'], params['qop'], a_2].join(":")
18
+ )
19
+
20
+ header = [
21
+ %Q(Digest username="#{user}"),
22
+ %Q(realm="#{params['realm']}"),
23
+ %Q(qop="#{params['qop']}"),
24
+ %Q(uri="#{@path}"),
25
+ %Q(nonce="#{params['nonce']}"),
26
+ %Q(nc="0"),
27
+ %Q(cnonce="#{params['cnonce']}"),
28
+ %Q(opaque="#{params['opaque']}"),
29
+ %Q(response="#{request_digest}")
30
+ ]
31
+
32
+ @header['Authorization'] = header
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,146 @@
1
+ module HTTPotato
2
+ class ParseError < StandardError; end
3
+ # The default parser used by HTTPotato, supports xml, json, html, yaml, and
4
+ # plain text.
5
+ #
6
+ # == Custom Parsers
7
+ #
8
+ # If you'd like to do your own custom parsing, subclassing HTTPotato::Parser
9
+ # will make that process much easier. There are a few different ways you can
10
+ # utilize HTTPotato::Parser as a superclass.
11
+ #
12
+ # @example Intercept the parsing for all formats
13
+ # class SimpleParser < HTTPotato::Parser
14
+ # def parse
15
+ # perform_parsing
16
+ # end
17
+ # end
18
+ #
19
+ # @example Add the atom format and parsing method to the default parser
20
+ # class AtomParsingIncluded < HTTPotato::Parser
21
+ # SupportedFormats.merge!(
22
+ # {"application/atom+xml" => :atom}
23
+ # )
24
+ #
25
+ # def atom
26
+ # perform_atom_parsing
27
+ # end
28
+ # end
29
+ #
30
+ # @example Only support the atom format
31
+ # class ParseOnlyAtom < HTTPotato::Parser
32
+ # SupportedFormats = {"application/atom+xml" => :atom}
33
+ #
34
+ # def atom
35
+ # perform_atom_parsing
36
+ # end
37
+ # end
38
+ #
39
+ # @abstract Read the Custom Parsers section for more information.
40
+ class Parser
41
+ SupportedFormats = {
42
+ 'text/xml' => :xml,
43
+ 'application/xml' => :xml,
44
+ 'application/json' => :json,
45
+ 'text/json' => :json,
46
+ 'application/javascript' => :json,
47
+ 'text/javascript' => :json,
48
+ 'text/html' => :html,
49
+ 'application/x-yaml' => :yaml,
50
+ 'text/yaml' => :yaml,
51
+ 'text/plain' => :plain
52
+ }
53
+
54
+ # The response body of the request
55
+ # @return [String]
56
+ attr_reader :body
57
+
58
+ # The intended parsing format for the request
59
+ # @return [Symbol] e.g. :json
60
+ attr_reader :format
61
+
62
+ # Instantiate the parser and call {#parse}.
63
+ # @param [String] body the response body
64
+ # @param [Symbol] format the response format
65
+ # @return parsed response
66
+ def self.call(body, format)
67
+ new(body, format).parse
68
+ end
69
+
70
+ # @return [Hash] the SupportedFormats hash
71
+ def self.formats
72
+ const_get(:SupportedFormats)
73
+ end
74
+
75
+ # @param [String] mimetype response MIME type
76
+ # @return [Symbol]
77
+ # @return [nil] mime type not supported
78
+ def self.format_from_mimetype(mimetype)
79
+ formats[formats.keys.detect {|k| mimetype.include?(k)}]
80
+ end
81
+
82
+ # @return [Array<Symbol>] list of supported formats
83
+ def self.supported_formats
84
+ formats.values.uniq
85
+ end
86
+
87
+ # @param [Symbol] format e.g. :json, :xml
88
+ # @return [Boolean]
89
+ def self.supports_format?(format)
90
+ supported_formats.include?(format)
91
+ end
92
+
93
+ def initialize(body, format)
94
+ @body = body
95
+ @format = format
96
+ end
97
+ private_class_method :new
98
+
99
+ # @return [Object] the parsed body
100
+ # @return [nil] when the response body is nil or an empty string
101
+ def parse
102
+ return nil if body.nil? || body.empty?
103
+ if supports_format?
104
+ parse_supported_format
105
+ else
106
+ body
107
+ end
108
+ end
109
+
110
+ protected
111
+
112
+ def xml
113
+ Crack::XML.parse(body)
114
+ end
115
+
116
+ def json
117
+ begin
118
+ JSON.parse(body)
119
+ rescue JSON::ParserError
120
+ raise ParseError, "Invalid JSON string #{body}"
121
+ end
122
+ end
123
+
124
+ def yaml
125
+ YAML.load(body)
126
+ end
127
+
128
+ def html
129
+ body
130
+ end
131
+
132
+ def plain
133
+ body
134
+ end
135
+
136
+ def supports_format?
137
+ self.class.supports_format?(format)
138
+ end
139
+
140
+ def parse_supported_format
141
+ send(format)
142
+ rescue NoMethodError
143
+ raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,231 @@
1
+ module HTTPotato
2
+ class Request #:nodoc:
3
+ SupportedHTTPMethods = [
4
+ Net::HTTP::Get,
5
+ Net::HTTP::Post,
6
+ Net::HTTP::Put,
7
+ Net::HTTP::Delete,
8
+ Net::HTTP::Head,
9
+ Net::HTTP::Options
10
+ ]
11
+
12
+ SupportedURISchemes = [URI::HTTP, URI::HTTPS]
13
+
14
+ attr_accessor :http_method, :path, :options, :last_response, :redirect
15
+
16
+ def initialize(http_method, path, o={})
17
+ self.http_method = http_method
18
+ self.path = path
19
+ self.options = {
20
+ :limit => o.delete(:no_follow) ? 1 : 5,
21
+ :default_params => {},
22
+ :parser => Parser
23
+ }.merge(o)
24
+ end
25
+
26
+ def path=(uri)
27
+ @path = URI.parse(uri)
28
+ end
29
+
30
+ def uri
31
+ new_uri = path.relative? ? URI.parse("#{options[:base_uri]}#{path}") : path
32
+
33
+ # avoid double query string on redirects [#12]
34
+ unless redirect
35
+ new_uri.query = query_string(new_uri)
36
+ end
37
+
38
+ unless SupportedURISchemes.include? new_uri.class
39
+ raise UnsupportedURIScheme, "'#{new_uri}' Must be HTTP or HTTPS"
40
+ end
41
+
42
+ new_uri
43
+ end
44
+
45
+ def format
46
+ options[:format] || (format_from_mimetype(last_response['content-type']) if last_response)
47
+ end
48
+
49
+ def parser
50
+ options[:parser]
51
+ end
52
+
53
+ def perform
54
+ validate
55
+ setup_raw_request
56
+ get_response
57
+ handle_response
58
+ end
59
+
60
+ private
61
+
62
+ def attach_ssl_certificates(http)
63
+ if http.use_ssl?
64
+ # Client certificate authentication
65
+ if options[:pem]
66
+ http.cert = OpenSSL::X509::Certificate.new(options[:pem])
67
+ http.key = OpenSSL::PKey::RSA.new(options[:pem])
68
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
69
+ end
70
+
71
+ # SSL certificate authority file and/or directory
72
+ if options[:ssl_ca_file]
73
+ http.ca_file = options[:ssl_ca_file]
74
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
75
+ end
76
+ if options[:ssl_ca_path]
77
+ http.ca_path = options[:ssl_ca_path]
78
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
79
+ end
80
+ end
81
+ end
82
+
83
+ def http
84
+ http = Net::HTTP.new(uri.host, uri.port, options[:http_proxyaddr], options[:http_proxyport])
85
+ http.use_ssl = ssl_implied?
86
+
87
+ if options[:timeout] && options[:timeout].is_a?(Integer)
88
+ http.open_timeout = options[:timeout]
89
+ http.read_timeout = options[:timeout]
90
+ end
91
+
92
+ attach_ssl_certificates(http)
93
+
94
+ if options[:debug_output]
95
+ http.set_debug_output(options[:debug_output])
96
+ end
97
+
98
+ http
99
+ end
100
+
101
+ def ssl_implied?
102
+ uri.port == 443 || uri.instance_of?(URI::HTTPS)
103
+ end
104
+
105
+ def body
106
+ options[:body].is_a?(Hash) ? options[:body].to_params : options[:body]
107
+ end
108
+
109
+ def credentials
110
+ options[:basic_auth] || options[:digest_auth]
111
+ end
112
+
113
+ def username
114
+ credentials[:username]
115
+ end
116
+
117
+ def password
118
+ credentials[:password]
119
+ end
120
+
121
+ def setup_raw_request
122
+ @raw_request = http_method.new(uri.request_uri)
123
+ @raw_request.body = body if body
124
+ @raw_request.initialize_http_header(options[:headers])
125
+ @raw_request.basic_auth(username, password) if options[:basic_auth]
126
+ AuthHMAC.sign!(@raw_request, options[:hmac][:id], options[:hmac][:secret]) if options[:hmac]
127
+ setup_digest_auth if options[:digest_auth]
128
+ end
129
+
130
+ def setup_digest_auth
131
+ res = http.head(uri.request_uri, options[:headers])
132
+ if res['www-authenticate'] != nil && res['www-authenticate'].length > 0
133
+ @raw_request.digest_auth(username, password, res)
134
+ end
135
+ end
136
+
137
+ def perform_actual_request
138
+ http.request(@raw_request)
139
+ end
140
+
141
+ def get_response
142
+ self.last_response = perform_actual_request
143
+ end
144
+
145
+ def query_string(uri)
146
+ query_string_parts = []
147
+ query_string_parts << uri.query unless uri.query.nil?
148
+
149
+ if options[:query].is_a?(Hash)
150
+ query_string_parts << options[:default_params].merge(options[:query]).to_params
151
+ else
152
+ query_string_parts << options[:default_params].to_params unless options[:default_params].empty?
153
+ query_string_parts << options[:query] unless options[:query].nil?
154
+ end
155
+
156
+ query_string_parts.size > 0 ? query_string_parts.join('&') : nil
157
+ end
158
+
159
+ # Raises exception Net::XXX (http error code) if an http error occured
160
+ def handle_response
161
+ handle_deflation
162
+ case last_response
163
+ when Net::HTTPMultipleChoice, # 300
164
+ Net::HTTPMovedPermanently, # 301
165
+ Net::HTTPFound, # 302
166
+ Net::HTTPSeeOther, # 303
167
+ Net::HTTPUseProxy, # 305
168
+ Net::HTTPTemporaryRedirect
169
+ if last_response.key?('location')
170
+ options[:limit] -= 1
171
+ self.path = last_response['location']
172
+ self.redirect = true
173
+ self.http_method = Net::HTTP::Get unless options[:maintain_method_across_redirects]
174
+ capture_cookies(last_response)
175
+ perform
176
+ else
177
+ last_response
178
+ end
179
+ else
180
+ Response.new(last_response, parse_response(last_response.body))
181
+ end
182
+ end
183
+
184
+ # Inspired by Ruby 1.9
185
+ def handle_deflation
186
+ case last_response["content-encoding"]
187
+ when "gzip"
188
+ body_io = StringIO.new(last_response.body)
189
+ last_response.body.replace Zlib::GzipReader.new(body_io).read
190
+ when "deflate"
191
+ last_response.body.replace Zlib::Inflate.inflate(last_response.body)
192
+ end
193
+ end
194
+
195
+ def parse_response(body)
196
+ parser.call(body, format)
197
+ end
198
+
199
+ def capture_cookies(response)
200
+ return unless response['Set-Cookie']
201
+ cookies_hash = HTTPotato::CookieHash.new()
202
+ cookies_hash.add_cookies(options[:headers]['Cookie']) if options[:headers] && options[:headers]['Cookie']
203
+ cookies_hash.add_cookies(response['Set-Cookie'])
204
+ options[:headers] ||= {}
205
+ options[:headers]['Cookie'] = cookies_hash.to_cookie_string
206
+ end
207
+
208
+ # Uses the HTTP Content-Type header to determine the format of the
209
+ # response It compares the MIME type returned to the types stored in the
210
+ # SupportedFormats hash
211
+ def format_from_mimetype(mimetype)
212
+ if mimetype && parser.respond_to?(:format_from_mimetype)
213
+ parser.format_from_mimetype(mimetype)
214
+ end
215
+ end
216
+
217
+ def validate
218
+ raise HTTPotato::RedirectionTooDeep.new(last_response), 'HTTP redirects too deep' if options[:limit].to_i <= 0
219
+ raise ArgumentError, 'only get, post, put, delete, head, and options methods are supported' unless SupportedHTTPMethods.include?(http_method)
220
+ raise ArgumentError, ':headers must be a hash' if options[:headers] && !options[:headers].is_a?(Hash)
221
+ raise ArgumentError, 'only one authentication method, :basic_auth or :digest_auth may be used at a time' if options[:basic_auth] && options[:digest_auth]
222
+ raise ArgumentError, ':basic_auth must be a hash' if options[:basic_auth] && !options[:basic_auth].is_a?(Hash)
223
+ raise ArgumentError, ':digest_auth must be a hash' if options[:digest_auth] && !options[:digest_auth].is_a?(Hash)
224
+ raise ArgumentError, ':query must be hash if using HTTP Post' if post? && !options[:query].nil? && !options[:query].is_a?(Hash)
225
+ end
226
+
227
+ def post?
228
+ Net::HTTP::Post == http_method
229
+ end
230
+ end
231
+ end