openlogic-resourceful 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. data/History.txt +45 -0
  2. data/MIT-LICENSE +21 -0
  3. data/Manifest +46 -0
  4. data/README.markdown +92 -0
  5. data/Rakefile +91 -0
  6. data/lib/resourceful.rb +27 -0
  7. data/lib/resourceful/abstract_form_data.rb +30 -0
  8. data/lib/resourceful/authentication_manager.rb +107 -0
  9. data/lib/resourceful/cache_manager.rb +242 -0
  10. data/lib/resourceful/exceptions.rb +34 -0
  11. data/lib/resourceful/header.rb +355 -0
  12. data/lib/resourceful/http_accessor.rb +103 -0
  13. data/lib/resourceful/memcache_cache_manager.rb +75 -0
  14. data/lib/resourceful/multipart_form_data.rb +46 -0
  15. data/lib/resourceful/net_http_adapter.rb +84 -0
  16. data/lib/resourceful/promiscuous_basic_authenticator.rb +18 -0
  17. data/lib/resourceful/request.rb +235 -0
  18. data/lib/resourceful/resource.rb +179 -0
  19. data/lib/resourceful/response.rb +221 -0
  20. data/lib/resourceful/simple.rb +36 -0
  21. data/lib/resourceful/stubbed_resource_proxy.rb +47 -0
  22. data/lib/resourceful/urlencoded_form_data.rb +19 -0
  23. data/lib/resourceful/util.rb +6 -0
  24. data/openlogic-resourceful.gemspec +51 -0
  25. data/resourceful.gemspec +51 -0
  26. data/spec/acceptance/authorization_spec.rb +16 -0
  27. data/spec/acceptance/caching_spec.rb +190 -0
  28. data/spec/acceptance/header_spec.rb +24 -0
  29. data/spec/acceptance/redirecting_spec.rb +12 -0
  30. data/spec/acceptance/resource_spec.rb +84 -0
  31. data/spec/acceptance/resourceful_spec.rb +56 -0
  32. data/spec/acceptance_shared_specs.rb +44 -0
  33. data/spec/caching_spec.rb +89 -0
  34. data/spec/old_acceptance_specs.rb +378 -0
  35. data/spec/resourceful/header_spec.rb +153 -0
  36. data/spec/resourceful/http_accessor_spec.rb +56 -0
  37. data/spec/resourceful/multipart_form_data_spec.rb +84 -0
  38. data/spec/resourceful/promiscuous_basic_authenticator_spec.rb +30 -0
  39. data/spec/resourceful/resource_spec.rb +20 -0
  40. data/spec/resourceful/response_spec.rb +51 -0
  41. data/spec/resourceful/urlencoded_form_data_spec.rb +64 -0
  42. data/spec/resourceful_spec.rb +79 -0
  43. data/spec/simple_sinatra_server.rb +74 -0
  44. data/spec/simple_sinatra_server_spec.rb +98 -0
  45. data/spec/spec.opts +3 -0
  46. data/spec/spec_helper.rb +31 -0
  47. metadata +192 -0
@@ -0,0 +1,103 @@
1
+ require 'net/http'
2
+
3
+ require 'resourceful/authentication_manager'
4
+ require 'resourceful/cache_manager'
5
+ require 'resourceful/resource'
6
+
7
+ require 'options'
8
+
9
+ module Resourceful
10
+ # This is an imitation Logger used when no real logger is
11
+ # registered. This allows most of the code to assume that there
12
+ # is always a logger available, which significantly improved the
13
+ # readability of the logging related code.
14
+ class BitBucketLogger
15
+ def warn(*args); end
16
+ def info(*args); end
17
+ def debug(*args); end
18
+ end
19
+
20
+ # This is the simplest logger. It just writes everything to STDOUT.
21
+ class StdOutLogger
22
+ def warn(*args); puts args; end
23
+ def info(*args); puts args; end
24
+ def debug(*args); puts args; end
25
+ end
26
+
27
+ # This class provides a simple interface to the functionality
28
+ # provided by the Resourceful library. Conceptually this object
29
+ # acts a collection of all the resources available via HTTP.
30
+ class HttpAccessor
31
+ # A logger object to which messages about the activities of this
32
+ # object will be written. This should be an object that responds
33
+ # to +#info(message)+ and +#debug(message)+.
34
+ #
35
+ # Errors will not be logged. Instead an exception will be raised
36
+ # and the application code should log it if appropriate.
37
+ attr_accessor :logger, :cache_manager
38
+
39
+ attr_reader :auth_manager
40
+ attr_reader :user_agent_tokens
41
+
42
+ ##
43
+ # The adapter this accessor will use to make the actual HTTP requests.
44
+ attr_reader :http_adapter
45
+
46
+ # Initializes a new HttpAccessor. Valid options:
47
+ #
48
+ # `:logger`
49
+ # : A Logger object that the new HTTP accessor should send log messages
50
+ #
51
+ # `:user_agent`
52
+ # : One or more additional user agent tokens to added to the user agent string.
53
+ #
54
+ # `:cache_manager`
55
+ # : The cache manager this accessor should use.
56
+ #
57
+ # `:authenticator`
58
+ # : Add a single authenticator for this accessor.
59
+ #
60
+ # `:authenticators`
61
+ # : Enumerable of the authenticators for this accessor.
62
+ #
63
+ # `http_adapter`
64
+ # : The HttpAdapter to be used by this accessor
65
+ #
66
+ #
67
+ def initialize(options = {})
68
+ options = Options.for(options).validate(:logger, :user_agent, :cache_manager, :authenticator, :authenticators, :http_adapter)
69
+
70
+ @user_agent_tokens = [RESOURCEFUL_USER_AGENT_TOKEN]
71
+ @auth_manager = AuthenticationManager.new()
72
+
73
+
74
+ @user_agent_tokens.push(*Array(options.getopt(:user_agent)).flatten.reverse)
75
+ self.logger = options.getopt(:logger) || BitBucketLogger.new
76
+ @cache_manager = options.getopt(:cache_manager) || NullCacheManager.new
77
+ @http_adapter = options.getopt(:http_adapter) || NetHttpAdapter.new
78
+
79
+ Array(options.getopt([:authenticator, :authenticators])).flatten.each do |an_authenticator|
80
+ add_authenticator(an_authenticator)
81
+ end
82
+ end
83
+
84
+ # Returns the string that identifies this HTTP accessor. If you
85
+ # want to add a token to the user agent string simply add the new
86
+ # token to the end of +#user_agent_tokens+.
87
+ def user_agent_string
88
+ user_agent_tokens.reverse.join(' ')
89
+ end
90
+
91
+ # Returns a resource object representing the resource indicated
92
+ # by the specified URI. A resource object will be created if necessary.
93
+ def resource(uri, opts = {})
94
+ resource = Resource.new(self, uri, opts)
95
+ end
96
+ alias [] resource
97
+
98
+ # Adds an Authenticator to the set used by the accessor.
99
+ def add_authenticator(an_authenticator)
100
+ auth_manager.add_auth_handler(an_authenticator)
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,75 @@
1
+ require "resourceful/cache_manager"
2
+
3
+ require 'memcache'
4
+
5
+ module Resourceful
6
+ class MemcacheCacheManager < AbstractCacheManager
7
+
8
+ # Create a new Memcached backed cache manager
9
+ #
10
+ # @param [*String] memcache_servers
11
+ # list of all Memcached servers this cache manager should use.
12
+ def initialize(*memcache_servers)
13
+ @memcache = MemCache.new(memcache_servers, :multithread => true)
14
+ end
15
+
16
+ # Finds a previously cached response to the provided request. The
17
+ # response returned may be stale.
18
+ #
19
+ # @param [Resourceful::Request] request
20
+ # The request for which we are looking for a response.
21
+ #
22
+ # @return [Resourceful::Response, nil]
23
+ # A (possibly stale) response for the request provided or nil if
24
+ # no matching response is found.
25
+ def lookup(request)
26
+ resp = cache_entries_for(request)[request]
27
+ return if resp.nil?
28
+
29
+ resp.authoritative = false
30
+
31
+ resp
32
+ end
33
+
34
+ # Store a response in the cache.
35
+ #
36
+ # This method is smart enough to not store responses that cannot be
37
+ # cached (Vary: * or Cache-Control: no-cache, private, ...)
38
+ #
39
+ # @param [Resourceful::Request] request
40
+ # The request used to obtain the response. This is needed so the
41
+ # values from the response's Vary header can be stored.
42
+ # @param [Resourceful::Response] response
43
+ # The response to be stored.
44
+ def store(request, response)
45
+ return unless response.cachable?
46
+
47
+ entries = cache_entries_for(request)
48
+ entries[request] = response
49
+
50
+ @memcache[request.to_mc_key] = entries
51
+ end
52
+
53
+ # Invalidates a all cached entries for a uri.
54
+ #
55
+ # This is used, for example, to invalidate the cache for a resource
56
+ # that gets POSTed to.
57
+ #
58
+ # @param [String] uri
59
+ # The uri of the resource to be invalidated
60
+ def invalidate(uri)
61
+ @memcache.delete(uri_hash(uri))
62
+ end
63
+
64
+
65
+ private
66
+
67
+ ##
68
+ # The memcache proxy.
69
+ attr_reader :memcache
70
+
71
+ def cache_entries_for(a_request)
72
+ @memcache.get(uri_hash(a_request.uri)) || Resourceful::CacheEntryCollection.new
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,46 @@
1
+ require 'resourceful/abstract_form_data'
2
+
3
+ module Resourceful
4
+ class MultipartFormData < AbstractFormData
5
+ FileParamValue = Struct.new(:content, :file_name, :content_type)
6
+
7
+ def add_file(name, file_name, content_type="application/octet-stream")
8
+ add(name, FileParamValue.new(File.new(file_name, 'r'), File.basename(file_name), content_type))
9
+ end
10
+
11
+ def content_type
12
+ "multipart/form-data; boundary=#{boundary}"
13
+ end
14
+
15
+ def read
16
+ StringIO.new.tap do |out|
17
+ first = true
18
+ form_data.each do |key, val|
19
+ out << "\r\n" unless first
20
+ out << "--" << boundary
21
+ out << "\r\nContent-Disposition: form-data; name=\"#{key}\""
22
+ if val.kind_of?(FileParamValue)
23
+ out << "; filename=\"#{val.file_name}\""
24
+ out << "\r\nContent-Type: #{val.content_type}"
25
+ end
26
+ out << "\r\n\r\n"
27
+ if val.kind_of?(FileParamValue)
28
+ out << val.content.read
29
+ else
30
+ out << val.to_s
31
+ end
32
+ first = false
33
+ end
34
+ out << "\r\n--#{boundary}--"
35
+ end.string
36
+ end
37
+
38
+ protected
39
+
40
+ def boundary
41
+ @boundary ||= (0..30).map{BOUNDARY_CHARS[rand(BOUNDARY_CHARS.length)]}.join
42
+ end
43
+
44
+ BOUNDARY_CHARS = [('a'..'z').to_a,('A'..'Z').to_a,(0..9).to_a].flatten
45
+ end
46
+ end
@@ -0,0 +1,84 @@
1
+ require 'net/http'
2
+ require 'net/https'
3
+ require 'addressable/uri'
4
+
5
+ require 'pathname'
6
+ require 'resourceful/header'
7
+
8
+ module Addressable
9
+ class URI
10
+ def absolute_path
11
+ absolute_path = ""
12
+ absolute_path << self.path.to_s
13
+ absolute_path << "?#{self.query}" if self.query != nil
14
+ absolute_path << "##{self.fragment}" if self.fragment != nil
15
+ return absolute_path
16
+ end
17
+ end
18
+ end
19
+
20
+ module Resourceful
21
+
22
+ class NetHttpAdapter
23
+ # Make an HTTP request using the standard library net/http.
24
+ #
25
+ # Will use a proxy defined in the http_proxy environment variable, if set.
26
+ #
27
+ # @param [#read] body
28
+ # An IO-ish thing containing the body of the request
29
+ #
30
+ def make_request(method, uri, body = nil, header = nil)
31
+ uri = uri.is_a?(Addressable::URI) ? uri : Addressable::URI.parse(uri)
32
+
33
+ if [:put, :post].include? method
34
+ body = body ? body.read : ""
35
+ header[:content_length] = body.size
36
+ end
37
+
38
+ req = net_http_request_class(method).new(uri.absolute_path)
39
+ header.each_field { |k,v| req[k] = v } if header
40
+ https = ("https" == uri.scheme)
41
+ conn_class = proxy_details ? Net::HTTP.Proxy(*proxy_details) : Net::HTTP
42
+ conn = conn_class.new(uri.host, uri.port || (https ? 443 : 80))
43
+ conn.use_ssl = https
44
+ begin
45
+ conn.start
46
+ res = if body
47
+ conn.request(req, body)
48
+ else
49
+ conn.request(req)
50
+ end
51
+ ensure
52
+ conn.finish if conn.started?
53
+ end
54
+
55
+ [ Integer(res.code),
56
+ Resourceful::Header.new(res.header.to_hash),
57
+ res.body
58
+ ]
59
+ ensure
60
+
61
+ end
62
+
63
+ private
64
+
65
+ # Parse proxy details from http_proxy environment variable
66
+ def proxy_details
67
+ proxy = Addressable::URI.parse(ENV["http_proxy"])
68
+ [proxy.host, proxy.port, proxy.user, proxy.password] if proxy
69
+ end
70
+
71
+ def net_http_request_class(method)
72
+ case method
73
+ when :get then Net::HTTP::Get
74
+ when :head then Net::HTTP::Head
75
+ when :post then Net::HTTP::Post
76
+ when :put then Net::HTTP::Put
77
+ when :delete then Net::HTTP::Delete
78
+ end
79
+
80
+ end
81
+
82
+ end
83
+
84
+ end
@@ -0,0 +1,18 @@
1
+ module Resourceful
2
+ # This class provides HTTP basic authentication without regard to
3
+ # the realm of receiving resource. This will send your username and
4
+ # password with any request made while it is in play.
5
+ class PromiscuousBasicAuthenticator < BasicAuthenticator
6
+ def initialize(username, password)
7
+ super(nil, username, password)
8
+ end
9
+
10
+ def valid_for?(challenge_response)
11
+ true
12
+ end
13
+
14
+ def can_handle?(request)
15
+ true
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,235 @@
1
+ require 'pathname'
2
+ require 'benchmark'
3
+ require 'resourceful/response'
4
+ require 'resourceful/net_http_adapter'
5
+ require 'resourceful/exceptions'
6
+
7
+ module Resourceful
8
+
9
+ class Request
10
+
11
+ REDIRECTABLE_METHODS = [:get, :head]
12
+ CACHEABLE_METHODS = [:get, :head]
13
+ INVALIDATING_METHODS = [:post, :put, :delete]
14
+
15
+ attr_accessor :method, :resource, :body, :header
16
+ attr_reader :request_time, :accessor
17
+
18
+ # @param [Symbol] http_method
19
+ # :get, :put, :post, :delete or :head
20
+ # @param [Resourceful::Resource] resource
21
+ # @param [#read, #rewind] body
22
+ # @param [Resourceful::Header, Hash] header
23
+ def initialize(http_method, resource, body = nil, header = nil)
24
+ @method, @resource, @body = http_method, resource, body
25
+ @accessor = @resource.accessor
26
+ @header = header.is_a?(Resourceful::Header) ? header : Resourceful::Header.new(header)
27
+
28
+ # Resourceful handled gzip encoding transparently, so set that up
29
+ @header.accept_encoding ||= 'gzip, identity'
30
+
31
+ # 'Host' is a required HTTP/1.1 header, overrides Host in user-provided headers
32
+ @header.host = @resource.host
33
+
34
+ # Setting the date isn't a bad idea, either
35
+ @header.date ||= Time.now.httpdate
36
+
37
+ # Add any auth credentials we might want
38
+ add_credentials!
39
+
40
+ end
41
+
42
+ # Uses the auth manager to add any valid credentials to this request
43
+ def add_credentials!
44
+ @accessor.auth_manager.add_credentials(self)
45
+ end
46
+
47
+ # Performs all the work. Handles caching, redirects, auth retries, etc
48
+ def fetch_response
49
+ if cached_response
50
+ if needs_revalidation?(cached_response)
51
+ logger.info(" Cache needs revalidation")
52
+ set_validation_headers!(cached_response)
53
+ else
54
+ # We're done!
55
+ return cached_response
56
+ end
57
+ end
58
+
59
+ response = perform!
60
+
61
+ response = revalidate_cached_response(response) if cached_response && response.not_modified?
62
+ response = follow_redirect(response) if should_be_redirected?(response)
63
+ response = retry_with_auth(response) if needs_authorization?(response)
64
+
65
+ raise UnsuccessfulHttpRequestError.new(self, response) if response.error?
66
+
67
+ if cacheable?(response)
68
+ store_in_cache(response)
69
+ elsif invalidates_cache?
70
+ invalidate_cache
71
+ end
72
+
73
+ return response
74
+ end
75
+
76
+ # Should we look for a response to this request in the cache?
77
+ def skip_cache?
78
+ return true unless method.in? CACHEABLE_METHODS
79
+ header.cache_control && header.cache_control.include?('no-cache')
80
+ end
81
+
82
+ # The cached response
83
+ def cached_response
84
+ return if skip_cache?
85
+ return if @cached_response.nil? && @already_checked_cache
86
+ @cached_response ||= begin
87
+ @already_checked_cache = true
88
+ resp = accessor.cache_manager.lookup(self)
89
+ logger.info(" Retrieved from cache") if resp
90
+ resp
91
+ end
92
+ end
93
+
94
+ # Revalidate the cached response with what we got from a 304 response
95
+ def revalidate_cached_response(not_modified_response)
96
+ logger.info(" Resource not modified")
97
+ cached_response.revalidate!(not_modified_response)
98
+ cached_response
99
+ end
100
+
101
+ # Follow a redirect response
102
+ def follow_redirect(response)
103
+ raise MalformedServerResponse.new(self, response) unless response.header.location
104
+ if response.moved_permanently?
105
+ new_uri = response.header.location.first
106
+ logger.info(" Permanently redirected to #{new_uri} - Storing new location.")
107
+ resource.update_uri new_uri
108
+ @header.host = resource.host
109
+ response = fetch_response
110
+ elsif response.see_other? # Always use GET for this redirect, regardless of initial method
111
+ redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
112
+ response = Request.new(:get, redirected_resource, body, header).fetch_response
113
+ else
114
+ redirected_resource = Resourceful::Resource.new(self.accessor, response.header['Location'].first)
115
+ logger.info(" Redirected to #{redirected_resource.uri} - Caching new location.")
116
+ response = Request.new(method, redirected_resource, body, header).fetch_response
117
+ end
118
+ end
119
+
120
+ # Add any auth headers from the response to the auth manager, and try the request again
121
+ def retry_with_auth(response)
122
+ @already_tried_with_auth = true
123
+ logger.info("Authentication Required. Retrying with auth info")
124
+ accessor.auth_manager.associate_auth_info(response)
125
+ add_credentials!
126
+ @body.rewind if @body # Its a stringIO, and we already fed it to the adapter once, so rewind it when we try again
127
+ response = fetch_response
128
+ end
129
+
130
+ # Does this request need to be authorized? Will only be true if we haven't already tried with auth
131
+ def needs_authorization?(response)
132
+ !@already_tried_with_auth && response.unauthorized?
133
+ end
134
+
135
+ # Store the response to this request in the cache
136
+ def store_in_cache(response)
137
+ # RFC2618 - 14.18 : A received message that does not have a Date header
138
+ # field MUST be assigned one by the recipient if the message will be cached
139
+ # by that recipient.
140
+ response.header.date ||= response.response_time.httpdate
141
+
142
+ accessor.cache_manager.store(self, response)
143
+ end
144
+
145
+ # Invalidated the cache for this uri (eg, after a POST)
146
+ def invalidate_cache
147
+ accessor.cache_manager.invalidate(resource.uri)
148
+ end
149
+
150
+ # Is this request & response permitted to be stored in this (private) cache?
151
+ def cacheable?(response)
152
+ return false unless response.success?
153
+ return false unless method.in? CACHEABLE_METHODS
154
+ return false if header.cache_control && header.cache_control.include?('no-store')
155
+ true
156
+ end
157
+
158
+ # Does this request invalidate the cache?
159
+ def invalidates_cache?
160
+ return true if method.in? INVALIDATING_METHODS
161
+ end
162
+
163
+ # Perform the request, with no magic handling of anything.
164
+ def perform!
165
+ @request_time = Time.now
166
+
167
+ http_resp = adapter.make_request(@method, @resource.uri, @body, @header)
168
+ @response = Resourceful::Response.new(uri, *http_resp)
169
+ @response.request_time = @request_time
170
+ @response.authoritative = true
171
+
172
+ @response
173
+ end
174
+
175
+ # Is this a response a redirect, and are we permitted to follow it?
176
+ def should_be_redirected?(response)
177
+ return false unless response.redirect?
178
+ if resource.on_redirect.nil?
179
+ return true if method.in? REDIRECTABLE_METHODS
180
+ false
181
+ else
182
+ resource.on_redirect.call(self, response)
183
+ end
184
+ end
185
+
186
+ # Do we need to revalidate our cache?
187
+ def needs_revalidation?(response)
188
+ return true if forces_revalidation?
189
+ return true if response.stale?
190
+ return true if max_age && response.current_age > max_age
191
+ return true if response.must_be_revalidated?
192
+ false
193
+ end
194
+
195
+ # Set the validation headers of a request based on the response in the cache
196
+ def set_validation_headers!(response)
197
+ @header['If-None-Match'] = response.header['ETag'] if response.header.has_key?('ETag')
198
+ @header['If-Modified-Since'] = response.header['Last-Modified'] if response.header.has_key?('Last-Modified')
199
+ @header['Cache-Control'] = 'max-age=0' if response.must_be_revalidated?
200
+ end
201
+
202
+ # @return [String] The URI against which this request will be, or was, made.
203
+ def uri
204
+ resource.uri
205
+ end
206
+
207
+ # Does this request force us to revalidate the cache?
208
+ def forces_revalidation?
209
+ if max_age == 0 || skip_cache?
210
+ logger.info(" Client forced revalidation")
211
+ true
212
+ else
213
+ false
214
+ end
215
+ end
216
+
217
+ # Indicates the maxmimum response age in seconds we are willing to accept
218
+ #
219
+ # Returns nil if we don't care how old the response is
220
+ def max_age
221
+ if header['Cache-Control'] and header['Cache-Control'].include?('max-age')
222
+ header['Cache-Control'].split(',').grep(/max-age/).first.split('=').last.to_i
223
+ end
224
+ end
225
+
226
+ def logger
227
+ resource.logger
228
+ end
229
+
230
+ def adapter
231
+ accessor.http_adapter
232
+ end
233
+ end
234
+
235
+ end