resourceful 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. data/Manifest +24 -28
  2. data/Rakefile +44 -14
  3. data/lib/resourceful.rb +11 -21
  4. data/lib/resourceful/authentication_manager.rb +3 -2
  5. data/lib/resourceful/cache_manager.rb +58 -1
  6. data/lib/resourceful/exceptions.rb +34 -0
  7. data/lib/resourceful/header.rb +95 -0
  8. data/lib/resourceful/http_accessor.rb +0 -2
  9. data/lib/resourceful/memcache_cache_manager.rb +3 -13
  10. data/lib/resourceful/net_http_adapter.rb +15 -5
  11. data/lib/resourceful/request.rb +180 -18
  12. data/lib/resourceful/resource.rb +38 -141
  13. data/lib/resourceful/response.rb +142 -95
  14. data/resourceful.gemspec +9 -7
  15. data/spec/acceptance/authorization_spec.rb +16 -0
  16. data/spec/acceptance/caching_spec.rb +192 -0
  17. data/spec/acceptance/header_spec.rb +24 -0
  18. data/spec/acceptance/redirecting_spec.rb +12 -0
  19. data/spec/acceptance/resource_spec.rb +84 -0
  20. data/spec/acceptance_shared_specs.rb +12 -17
  21. data/spec/{acceptance_spec.rb → old_acceptance_specs.rb} +27 -57
  22. data/spec/simple_sinatra_server.rb +74 -0
  23. data/spec/simple_sinatra_server_spec.rb +98 -0
  24. data/spec/spec_helper.rb +21 -7
  25. metadata +50 -42
  26. data/spec/resourceful/authentication_manager_spec.rb +0 -249
  27. data/spec/resourceful/cache_manager_spec.rb +0 -223
  28. data/spec/resourceful/header_spec.rb +0 -38
  29. data/spec/resourceful/http_accessor_spec.rb +0 -164
  30. data/spec/resourceful/memcache_cache_manager_spec.rb +0 -111
  31. data/spec/resourceful/net_http_adapter_spec.rb +0 -96
  32. data/spec/resourceful/options_interpreter_spec.rb +0 -102
  33. data/spec/resourceful/request_spec.rb +0 -186
  34. data/spec/resourceful/resource_spec.rb +0 -600
  35. data/spec/resourceful/response_spec.rb +0 -238
  36. data/spec/resourceful/stubbed_resource_proxy_spec.rb +0 -58
  37. data/spec/simple_http_server_shared_spec.rb +0 -162
  38. data/spec/simple_http_server_shared_spec_spec.rb +0 -212
@@ -3,7 +3,7 @@ require 'net/https'
3
3
  require 'addressable/uri'
4
4
 
5
5
  require 'pathname'
6
- require Pathname(__FILE__).dirname + 'header'
6
+ require 'resourceful/header'
7
7
 
8
8
  module Addressable
9
9
  class URI
@@ -20,18 +20,22 @@ end
20
20
  module Resourceful
21
21
 
22
22
  class NetHttpAdapter
23
+ # Make an HTTP request using the standard library net/http.
24
+ #
25
+ # Will use a proxy defined in the http_proxy environment variable, if set.
23
26
  def self.make_request(method, uri, body = nil, header = nil)
24
27
  uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
25
28
 
26
29
  req = net_http_request_class(method).new(uri.absolute_path)
27
- header.each { |k,v| req[k] = v } if header
28
- conn = Net::HTTP.new(uri.host, uri.port)
29
- conn.use_ssl = (/https/i === uri.scheme)
30
+ header.each_field { |k,v| req[k] = v } if header
31
+ https = ("https" == uri.scheme)
32
+ conn = Net::HTTP.Proxy(*proxy_details).new(uri.host, uri.port || (https ? 443 : 80))
33
+ conn.use_ssl = https
30
34
  begin
31
35
  conn.start
32
36
  res = conn.request(req, body)
33
37
  ensure
34
- conn.finish
38
+ conn.finish if conn.started?
35
39
  end
36
40
 
37
41
  [ Integer(res.code),
@@ -44,6 +48,12 @@ module Resourceful
44
48
 
45
49
  private
46
50
 
51
+ # Parse proxy details from http_proxy environment variable
52
+ def self.proxy_details
53
+ proxy = Addressable::URI.parse(ENV["http_proxy"])
54
+ [proxy.host, proxy.port, proxy.user, proxy.password] if proxy
55
+ end
56
+
47
57
  def self.net_http_request_class(method)
48
58
  case method
49
59
  when :get then Net::HTTP::Get
@@ -1,16 +1,19 @@
1
1
  require 'pathname'
2
2
  require 'benchmark'
3
- require Pathname(__FILE__).dirname + 'response'
4
- require Pathname(__FILE__).dirname + 'net_http_adapter'
3
+ require 'resourceful/response'
4
+ require 'resourceful/net_http_adapter'
5
+ require 'resourceful/exceptions'
5
6
 
6
7
  module Resourceful
7
8
 
8
9
  class Request
9
10
 
10
11
  REDIRECTABLE_METHODS = [:get, :head]
12
+ CACHEABLE_METHODS = [:get, :head]
13
+ INVALIDATING_METHODS = [:post, :put, :delete]
11
14
 
12
15
  attr_accessor :method, :resource, :body, :header
13
- attr_reader :request_time
16
+ attr_reader :request_time, :accessor
14
17
 
15
18
  # @param [Symbol] http_method
16
19
  # :get, :put, :post, :delete or :head
@@ -19,28 +22,159 @@ module Resourceful
19
22
  # @param [Resourceful::Header, Hash] header
20
23
  def initialize(http_method, resource, body = nil, header = nil)
21
24
  @method, @resource, @body = http_method, resource, body
22
- @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header || {})
25
+ @accessor = @resource.accessor
26
+ @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header)
23
27
 
24
- @header['Accept-Encoding'] = 'gzip, identity'
25
- # 'Host' is a required HTTP/1.1 header, so set it if it isn't already
26
- @header['Host'] ||= Addressable::URI.parse(resource.uri).host
28
+ # Resourceful handled gzip encoding transparently, so set that up
29
+ @header.accept_encoding ||= 'gzip, identity'
30
+
31
+ # 'Host' is a required HTTP/1.1 header, overrides Host in user-provided headers
32
+ @header.host = @resource.host
27
33
 
28
34
  # Setting the date isn't a bad idea, either
29
- @header['Date'] ||= Time.now.httpdate
35
+ @header.date ||= Time.now.httpdate
36
+
37
+ # Add any auth credentials we might want
38
+ add_credentials!
39
+
40
+ end
41
+
42
+ # Uses the auth manager to add any valid credentials to this request
43
+ def add_credentials!
44
+ @accessor.auth_manager.add_credentials(self)
45
+ end
46
+
47
+ # Performs all the work. Handles caching, redirects, auth retries, etc
48
+ def fetch_response
49
+ if cached_response
50
+ if needs_revalidation?(cached_response)
51
+ logger.info(" Cache needs revalidation")
52
+ set_validation_headers!(cached_response)
53
+ else
54
+ # We're done!
55
+ return cached_response
56
+ end
57
+ end
58
+
59
+ response = perform!
60
+
61
+ response = revalidate_cached_response(response) if cached_response && response.not_modified?
62
+ response = follow_redirect(response) if should_be_redirected?(response)
63
+ response = retry_with_auth(response) if needs_authorization?(response)
64
+
65
+ raise UnsuccessfulHttpRequestError.new(self, response) if response.error?
66
+
67
+ if cacheable?(response)
68
+ store_in_cache(response)
69
+ elsif invalidates_cache?
70
+ invalidate_cache
71
+ end
72
+
73
+ return response
74
+ end
75
+
76
+ # Should we look for a response to this request in the cache?
77
+ def skip_cache?
78
+ return true unless method.in? CACHEABLE_METHODS
79
+ header.cache_control && header.cache_control.include?('no-cache')
80
+ end
81
+
82
+ # The cached response
83
+ def cached_response
84
+ return if skip_cache?
85
+ return if @cached_response.nil? && @already_checked_cache
86
+ @cached_response ||= begin
87
+ @already_checked_cache = true
88
+ resp = accessor.cache_manager.lookup(self)
89
+ logger.info(" Retrieved from cache")
90
+ resp
91
+ end
92
+ end
93
+
94
+ # Revalidate the cached response with what we got from a 304 response
95
+ def revalidate_cached_response(not_modified_response)
96
+ logger.info(" Resource not modified")
97
+ cached_response.revalidate!(not_modified_response)
98
+ cached_response
99
+ end
100
+
101
+ # Follow a redirect response
102
+ def follow_redirect(response)
103
+ raise MalformedServerResponse.new(self, response) unless response.header.location
104
+ if response.moved_permanently?
105
+ new_uri = response.header.location.first
106
+ logger.info(" Permanently redirected to #{new_uri} - Storing new location.")
107
+ resource.update_uri new_uri
108
+ @header.host = resource.host
109
+ response = fetch_response
110
+ elsif response.see_other? # Always use GET for this redirect, regardless of initial method
111
+ redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
112
+ response = Request.new(:get, redirected_resource, body, header).fetch_response
113
+ else
114
+ redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
115
+ logger.info(" Redirected to #{redirected_resource.uri} - Caching new location.")
116
+ response = Request.new(method, redirected_resource, body, header).fetch_response
117
+ end
30
118
  end
31
119
 
32
- def response
120
+ # Add any auth headers from the response to the auth manager, and try the request again
121
+ def retry_with_auth(response)
122
+ @already_tried_with_auth = true
123
+ logger.info("Authentication Required. Retrying with auth info")
124
+ accessor.auth_manager.associate_auth_info(response)
125
+ add_credentials!
126
+ response = fetch_response
127
+ end
128
+
129
+ # Does this request need to be authorized? Will only be true if we haven't already tried with auth
130
+ def needs_authorization?(response)
131
+ !@already_tried_with_auth && response.unauthorized?
132
+ end
133
+
134
+ # Store the response to this request in the cache
135
+ def store_in_cache(response)
136
+ # RFC2618 - 14.18 : A received message that does not have a Date header
137
+ # field MUST be assigned one by the recipient if the message will be cached
138
+ # by that recipient.
139
+ response.header.date ||= response.response_time.httpdate
140
+
141
+ accessor.cache_manager.store(self, response)
142
+ end
143
+
144
+ # Invalidated the cache for this uri (eg, after a POST)
145
+ def invalidate_cache
146
+ accessor.cache_manager.invalidate(resource.uri)
147
+ end
148
+
149
+ # Is this request & response permitted to be stored in this (private) cache?
150
+ def cacheable?(response)
151
+ return false unless response.success?
152
+ return false unless method.in? CACHEABLE_METHODS
153
+ return false if header.cache_control && header.cache_control.include?('no-store')
154
+ true
155
+ end
156
+
157
+ # Does this request invalidate the cache?
158
+ def invalidates_cache?
159
+ return true if method.in? INVALIDATING_METHODS
160
+ end
161
+
162
+ # Perform the request, with no magic handling of anything.
163
+ def perform!
33
164
  @request_time = Time.now
165
+ logger.debug("DEBUG: Request Header: #{@header.inspect}")
34
166
 
35
167
  http_resp = NetHttpAdapter.make_request(@method, @resource.uri, @body, @header)
36
- response = Resourceful::Response.new(uri, *http_resp)
37
- response.request_time = @request_time
168
+ @response = Resourceful::Response.new(uri, *http_resp)
169
+ @response.request_time = @request_time
170
+ @response.authoritative = true
38
171
 
39
- response.authoritative = true
40
- response
172
+ @response
41
173
  end
42
174
 
43
- def should_be_redirected?
175
+ # Is this a response a redirect, and are we permitted to follow it?
176
+ def should_be_redirected?(response)
177
+ return false unless response.redirect?
44
178
  if resource.on_redirect.nil?
45
179
  return true if method.in? REDIRECTABLE_METHODS
46
180
  false
@@ -49,21 +183,49 @@ module Resourceful
49
183
  end
50
184
  end
51
185
 
52
- def set_validation_headers(response)
186
+ # Do we need to revalidate our cache?
187
+ def needs_revalidation?(response)
188
+ return true if forces_revalidation?
189
+ return true if response.stale?
190
+ return true if max_age && response.current_age > max_age
191
+ return true if response.must_be_revalidated?
192
+ false
193
+ end
194
+
195
+ # Set the validation headers of a request based on the response in the cache
196
+ def set_validation_headers!(response)
53
197
  @header['If-None-Match'] = response.header['ETag'] if response.header.has_key?('ETag')
54
198
  @header['If-Modified-Since'] = response.header['Last-Modified'] if response.header.has_key?('Last-Modified')
55
- @header['Cache-Control'] = 'max-age=0' if response.header.has_key?('Cache-Control') and response.header['Cache-Control'].include?('must-revalidate')
199
+ @header['Cache-Control'] = 'max-age=0' if response.must_be_revalidated?
56
200
  end
57
201
 
58
202
  # @return [String] The URI against which this request will be, or was, made.
59
203
  def uri
60
204
  resource.uri
61
205
  end
62
-
206
+
207
+ # Does this request force us to revalidate the cache?
208
+ def forces_revalidation?
209
+ if max_age == 0 || header.cache_control && cc.include?('no-cache')
210
+ logger.info(" Client forced revalidation")
211
+ true
212
+ else
213
+ false
214
+ end
215
+ end
216
+
217
+ # Indicates the maxmimum response age in seconds we are willing to accept
218
+ #
219
+ # Returns nil if we don't care how old the response is
220
+ def max_age
221
+ if header['Cache-Control'] and header['Cache-Control'].include?('max-age')
222
+ header['Cache-Control'].split(',').grep(/max-age/).first.split('=').last.to_i
223
+ end
224
+ end
225
+
63
226
  def logger
64
227
  resource.logger
65
228
  end
66
-
67
229
  end
68
230
 
69
231
  end
@@ -1,25 +1,11 @@
1
1
  require 'pathname'
2
- require Pathname(__FILE__).dirname + 'request'
2
+ require 'resourceful/request'
3
3
 
4
4
  module Resourceful
5
5
 
6
- # This exception used to indicate that the request did not succeed.
7
- # The HTTP response is included so that the appropriate actions can
8
- # be taken based on the details of that response
9
- class UnsuccessfulHttpRequestError < Exception
10
- attr_reader :http_response, :http_request
11
-
12
- # Initialize new error from the HTTP request and response attributes.
13
- def initialize(http_request, http_response)
14
- super("#{http_request.method} request to <#{http_request.uri}> failed with code #{http_response.code}")
15
- @http_request = http_request
16
- @http_response = http_response
17
- end
18
- end
19
-
20
6
  class Resource
21
7
  attr_reader :accessor
22
- attr_accessor :default_options
8
+ attr_accessor :default_header
23
9
 
24
10
  # Build a new resource for a uri
25
11
  #
@@ -27,9 +13,9 @@ module Resourceful
27
13
  # The parent http accessor
28
14
  # @param uri<String, Addressable::URI>
29
15
  # The uri for the location of the resource
30
- def initialize(accessor, uri, options = {})
16
+ def initialize(accessor, uri, default_header = {})
31
17
  @accessor, @uris = accessor, [uri]
32
- @default_options = options
18
+ @default_header = Resourceful::Header.new({'User-Agent' => Resourceful::RESOURCEFUL_USER_AGENT_TOKEN}.merge(default_header))
33
19
  @on_redirect = nil
34
20
  end
35
21
 
@@ -44,6 +30,16 @@ module Resourceful
44
30
  end
45
31
  alias uri effective_uri
46
32
 
33
+ # Returns the host for this Resource's current uri
34
+ def host
35
+ Addressable::URI.parse(uri).host
36
+ end
37
+
38
+ # Updates the effective uri after following a permanent redirect
39
+ def update_uri(uri)
40
+ @uris.unshift(uri)
41
+ end
42
+
47
43
  # When performing a redirect, this callback will be executed first. If the callback
48
44
  # returns true, then the redirect is followed, otherwise it is not. The request that
49
45
  # triggered the redirect and the response will be passed into the block. This can be
@@ -76,9 +72,7 @@ module Resourceful
76
72
  # @raise [UnsuccessfulHttpRequestError] unless the request is a
77
73
  # success, ie the final request returned a 2xx response code
78
74
  def get(header = {})
79
- log_request_with_time "GET [#{uri}]" do
80
- do_read_request(:get, header)
81
- end
75
+ request(:get, nil, header)
82
76
  end
83
77
 
84
78
  # :call-seq:
@@ -97,12 +91,9 @@ module Resourceful
97
91
  # @raise [ArgumentError] unless :content-type is specified in options
98
92
  # @raise [UnsuccessfulHttpRequestError] unless the request is a
99
93
  # success, ie the final request returned a 2xx response code
100
- def post(data = "", options = {})
101
- raise ArgumentError, ":content_type must be specified" unless options.has_key?(:content_type)
102
-
103
- log_request_with_time "POST [#{uri}]" do
104
- do_write_request(:post, data, options)
105
- end
94
+ def post(data = nil, header = {})
95
+ check_content_type_exists(data, header)
96
+ request(:post, data, header)
106
97
  end
107
98
 
108
99
  # :call-seq:
@@ -121,12 +112,9 @@ module Resourceful
121
112
  # @raise [ArgumentError] unless :content-type is specified in options
122
113
  # @raise [UnsuccessfulHttpRequestError] unless the request is a
123
114
  # success, ie the final request returned a 2xx response code
124
- def put(data = "", options = {})
125
- raise ArgumentError, ":content_type must be specified" unless options.has_key?(:content_type)
126
-
127
- log_request_with_time "PUT [#{uri}]" do
128
- do_write_request(:put, data, options)
129
- end
115
+ def put(data, header = {})
116
+ check_content_type_exists(data, header)
117
+ request(:put, data, header)
130
118
  end
131
119
 
132
120
  # Performs a DELETE on the resource, following redirects as neccessary.
@@ -135,120 +123,33 @@ module Resourceful
135
123
  #
136
124
  # @raise [UnsuccessfulHttpRequestError] unless the request is a
137
125
  # success, ie the final request returned a 2xx response code
138
- def delete(options = {})
139
- log_request_with_time "DELETE [#{uri}]" do
140
- do_write_request(:delete, {}, options)
141
- end
126
+ def delete(header = {})
127
+ request(:delete, nil, header)
142
128
  end
143
129
 
144
- # Performs a read request (HEAD, GET). Users should use the #get, etc methods instead.
145
- #
146
- # This method handles all the work of following redirects.
147
- #
148
- # @param method<Symbol> The method to perform
149
- #
150
- # @return <Response>
151
- #
152
- # @raise [UnsuccessfulHttpRequestError] unless the request is a
153
- # success, ie the final request returned a 2xx response code
154
- #
155
- def do_read_request(method, header = {})
156
- request = Resourceful::Request.new(method, self, nil, default_options.merge(header))
157
- accessor.auth_manager.add_credentials(request)
158
-
159
- cached_response = accessor.cache_manager.lookup(request)
160
- if cached_response
161
- logger.info(" Retrieved from cache")
162
- if not cached_response.stale?
163
- # We're done!
164
- return cached_response
165
- else
166
- logger.info(" Cache entry is stale")
167
- request.set_validation_headers(cached_response)
168
- end
169
- end
170
-
171
- response = request.response
172
-
173
- if response.is_not_modified?
174
- logger.info(" Resource not modified")
175
- cached_response.header.merge!(response.header)
176
- cached_response.request_time = response.request_time
177
- response = cached_response
178
- response.authoritative = true
179
- end
180
-
181
- if response.is_redirect? and request.should_be_redirected?
182
- if response.is_permanent_redirect?
183
- @uris.unshift response.header['Location'].first
184
- logger.info(" Permanently redirected to #{uri} - Storing new location.")
185
- response = do_read_request(method, header)
186
- else
187
- redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
188
- logger.info(" Redirected to #{redirected_resource.uri} - Storing new location.")
189
- response = redirected_resource.do_read_request(method, header)
190
- end
191
- end
192
-
193
- if response.is_not_authorized? && !@already_tried_with_auth
194
- @already_tried_with_auth = true
195
- accessor.auth_manager.associate_auth_info(response)
196
- logger.info("Authentication Required. Retrying with auth info")
197
- response = do_read_request(method, header)
198
- end
199
-
200
- raise UnsuccessfulHttpRequestError.new(request,response) unless response.is_success?
201
-
202
- accessor.cache_manager.store(request, response) if response.is_success?
203
-
204
- return response
130
+ def logger
131
+ accessor.logger
205
132
  end
206
133
 
207
- # Performs a write request (POST, PUT, DELETE). Users should use the #post, etc
208
- # methods instead.
209
- #
210
- # This method handles all the work of following redirects.
211
- #
212
- # @param [Symbol] method The method to perform
213
- # @param [String] data Body of the http request.
214
- # @param [Hash] header Header for the HTTP resquest.
215
- #
216
- # @return [Response]
217
- #
218
- # @raise [UnsuccessfulHttpRequestError] unless the request is a
219
- # success, ie the final request returned a 2xx response code
220
- def do_write_request(method, data = nil, header = {})
221
- request = Resourceful::Request.new(method, self, data, default_options.merge(header))
222
- accessor.auth_manager.add_credentials(request)
223
-
224
- response = request.response
134
+ private
225
135
 
226
- if response.is_redirect? and request.should_be_redirected?
227
- if response.is_permanent_redirect?
228
- @uris.unshift response.header['Location'].first
229
- response = do_write_request(method, data, header)
230
- elsif response.code == 303 # see other, must use GET for new location
231
- redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
232
- response = redirected_resource.do_read_request(:get, header)
233
- else
234
- redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
235
- response = redirected_resource.do_write_request(method, data, header)
236
- end
136
+ # Ensures that the request has a content type header
137
+ # TODO Move this to request
138
+ def check_content_type_exists(body, header)
139
+ if body
140
+ raise MissingContentType unless header.has_key?(:content_type) or default_header.has_key?(:content_type)
237
141
  end
142
+ end
238
143
 
239
- if response.is_not_authorized? && !@already_tried_with_auth
240
- @already_tried_with_auth = true
241
- accessor.auth_manager.associate_auth_info(response)
242
- logger.debug("Authentication Required. Retrying with auth info")
243
- response = do_write_request(method, data, header)
144
+ # Actually make the request
145
+ def request(method, data, header)
146
+ log_request_with_time "#{method.to_s.upcase} [#{uri}]" do
147
+ request = Request.new(method, self, data, default_header.merge(header))
148
+ request.fetch_response
244
149
  end
245
-
246
- raise UnsuccessfulHttpRequestError.new(request,response) unless response.is_success?
247
-
248
- accessor.cache_manager.invalidate(uri)
249
- return response
250
150
  end
251
151
 
152
+ # Log it took the time to make the request
252
153
  def log_request_with_time(msg, indent = 2)
253
154
  logger.info(" " * indent + msg)
254
155
  result = nil
@@ -257,10 +158,6 @@ module Resourceful
257
158
  result
258
159
  end
259
160
 
260
- def logger
261
- accessor.logger
262
- end
263
-
264
161
  end
265
162
 
266
163
  end