http 0.7.4 → 0.8.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/.rspec +0 -1
  3. data/.rubocop.yml +5 -2
  4. data/CHANGES.md +24 -7
  5. data/CONTRIBUTING.md +25 -0
  6. data/Gemfile +24 -22
  7. data/Guardfile +2 -2
  8. data/README.md +34 -4
  9. data/Rakefile +7 -7
  10. data/examples/parallel_requests_with_celluloid.rb +2 -2
  11. data/http.gemspec +12 -12
  12. data/lib/http.rb +11 -10
  13. data/lib/http/cache.rb +146 -0
  14. data/lib/http/cache/headers.rb +100 -0
  15. data/lib/http/cache/null_cache.rb +13 -0
  16. data/lib/http/chainable.rb +14 -3
  17. data/lib/http/client.rb +64 -80
  18. data/lib/http/connection.rb +139 -0
  19. data/lib/http/content_type.rb +2 -2
  20. data/lib/http/errors.rb +7 -1
  21. data/lib/http/headers.rb +21 -8
  22. data/lib/http/headers/mixin.rb +1 -1
  23. data/lib/http/mime_type.rb +2 -2
  24. data/lib/http/mime_type/adapter.rb +2 -2
  25. data/lib/http/mime_type/json.rb +4 -4
  26. data/lib/http/options.rb +65 -74
  27. data/lib/http/redirector.rb +3 -3
  28. data/lib/http/request.rb +20 -13
  29. data/lib/http/request/caching.rb +95 -0
  30. data/lib/http/request/writer.rb +5 -5
  31. data/lib/http/response.rb +15 -9
  32. data/lib/http/response/body.rb +21 -8
  33. data/lib/http/response/caching.rb +142 -0
  34. data/lib/http/response/io_body.rb +63 -0
  35. data/lib/http/response/parser.rb +1 -1
  36. data/lib/http/response/status.rb +4 -12
  37. data/lib/http/response/status/reasons.rb +53 -53
  38. data/lib/http/response/string_body.rb +53 -0
  39. data/lib/http/version.rb +1 -1
  40. data/spec/lib/http/cache/headers_spec.rb +77 -0
  41. data/spec/lib/http/cache_spec.rb +182 -0
  42. data/spec/lib/http/client_spec.rb +123 -95
  43. data/spec/lib/http/content_type_spec.rb +25 -25
  44. data/spec/lib/http/headers/mixin_spec.rb +8 -8
  45. data/spec/lib/http/headers_spec.rb +213 -173
  46. data/spec/lib/http/options/body_spec.rb +5 -5
  47. data/spec/lib/http/options/form_spec.rb +3 -3
  48. data/spec/lib/http/options/headers_spec.rb +7 -7
  49. data/spec/lib/http/options/json_spec.rb +3 -3
  50. data/spec/lib/http/options/merge_spec.rb +26 -22
  51. data/spec/lib/http/options/new_spec.rb +10 -10
  52. data/spec/lib/http/options/proxy_spec.rb +8 -8
  53. data/spec/lib/http/options_spec.rb +2 -2
  54. data/spec/lib/http/redirector_spec.rb +32 -32
  55. data/spec/lib/http/request/caching_spec.rb +133 -0
  56. data/spec/lib/http/request/writer_spec.rb +26 -26
  57. data/spec/lib/http/request_spec.rb +63 -58
  58. data/spec/lib/http/response/body_spec.rb +13 -13
  59. data/spec/lib/http/response/caching_spec.rb +201 -0
  60. data/spec/lib/http/response/io_body_spec.rb +35 -0
  61. data/spec/lib/http/response/status_spec.rb +25 -25
  62. data/spec/lib/http/response/string_body_spec.rb +35 -0
  63. data/spec/lib/http/response_spec.rb +64 -45
  64. data/spec/lib/http_spec.rb +103 -76
  65. data/spec/spec_helper.rb +10 -12
  66. data/spec/support/connection_reuse_shared.rb +100 -0
  67. data/spec/support/create_certs.rb +12 -12
  68. data/spec/support/dummy_server.rb +11 -11
  69. data/spec/support/dummy_server/servlet.rb +43 -31
  70. data/spec/support/proxy_server.rb +31 -25
  71. metadata +57 -8
  72. data/spec/support/example_server.rb +0 -30
  73. data/spec/support/example_server/servlet.rb +0 -102
@@ -9,7 +9,7 @@ module HTTP
9
9
  new mime_type(str), charset(str)
10
10
  end
11
11
 
12
- private
12
+ private
13
13
 
14
14
  # :nodoc:
15
15
  def mime_type(str)
@@ -20,7 +20,7 @@ module HTTP
20
20
  # :nodoc:
21
21
  def charset(str)
22
22
  md = str.to_s.match CHARSET_RE
23
- md && md[1].to_s.strip.gsub(/^"|"$/, '')
23
+ md && md[1].to_s.strip.gsub(/^"|"$/, "")
24
24
  end
25
25
  end
26
26
  end
data/lib/http/errors.rb CHANGED
@@ -8,6 +8,12 @@ module HTTP
8
8
  # Generic Response error
9
9
  class ResponseError < Error; end
10
10
 
11
- # Request to do something when we're in the wrong state
11
+ # Requested to do something when we're in the wrong state
12
12
  class StateError < ResponseError; end
13
+
14
+ # Generic Cache error
15
+ class CacheError < Error; end
16
+
17
+ # Header name is invalid
18
+ class InvalidHeaderNameError < Error; end
13
19
  end
data/lib/http/headers.rb CHANGED
@@ -1,6 +1,7 @@
1
- require 'forwardable'
1
+ require "forwardable"
2
2
 
3
- require 'http/headers/mixin'
3
+ require "http/errors"
4
+ require "http/headers/mixin"
4
5
 
5
6
  module HTTP
6
7
  # HTTP Headers container.
@@ -11,6 +12,10 @@ module HTTP
11
12
  # Matches HTTP header names when in "Canonical-Http-Format"
12
13
  CANONICAL_HEADER = /^[A-Z][a-z]*(-[A-Z][a-z]*)*$/
13
14
 
15
+ # Matches valid header field name according to RFC.
16
+ # @see http://tools.ietf.org/html/rfc7230#section-3.2
17
+ HEADER_NAME_RE = /^[A-Za-z0-9!#\$%&'*+\-.^_`|~]+$/
18
+
14
19
  # Class constructor.
15
20
  def initialize
16
21
  @pile = []
@@ -31,7 +36,7 @@ module HTTP
31
36
  # @param [#to_s] name header name
32
37
  # @return [void]
33
38
  def delete(name)
34
- name = canonicalize_header name.to_s
39
+ name = normalize_header name.to_s
35
40
  @pile.delete_if { |k, _| k == name }
36
41
  end
37
42
 
@@ -41,7 +46,7 @@ module HTTP
41
46
  # @param [Array<#to_s>, #to_s] value header value(s) to be appended
42
47
  # @return [void]
43
48
  def add(name, value)
44
- name = canonicalize_header name.to_s
49
+ name = normalize_header name.to_s
45
50
  Array(value).each { |v| @pile << [name, v.to_s] }
46
51
  end
47
52
 
@@ -52,7 +57,7 @@ module HTTP
52
57
  #
53
58
  # @return [Array<String>]
54
59
  def get(name)
55
- name = canonicalize_header name.to_s
60
+ name = normalize_header name.to_s
56
61
  @pile.select { |k, _| k == name }.map { |_, v| v }
57
62
  end
58
63
 
@@ -77,6 +82,7 @@ module HTTP
77
82
  def to_h
78
83
  Hash[keys.map { |k| [k, self[k]] }]
79
84
  end
85
+ alias_method :to_hash, :to_h
80
86
 
81
87
  # Returns headers key/value pairs.
82
88
  #
@@ -179,14 +185,21 @@ module HTTP
179
185
  alias_method :[], :coerce
180
186
  end
181
187
 
182
- private
188
+ private
183
189
 
184
190
  # Transforms `name` to canonical HTTP header capitalization
185
191
  #
186
192
  # @param [String] name
193
+ # @raise [InvalidHeaderNameError] if normalized name does not
194
+ # match {HEADER_NAME_RE}
187
195
  # @return [String] canonical HTTP header name
188
- def canonicalize_header(name)
189
- name[CANONICAL_HEADER] || name.split(/[\-_]/).map(&:capitalize).join('-')
196
+ def normalize_header(name)
197
+ normalized = name[CANONICAL_HEADER]
198
+ normalized ||= name.split(/[\-_]/).map(&:capitalize).join("-")
199
+
200
+ return normalized if normalized =~ HEADER_NAME_RE
201
+
202
+ fail InvalidHeaderNameError, "Invalid HTTP header field name: #{name.inspect}"
190
203
  end
191
204
  end
192
205
  end
@@ -1,4 +1,4 @@
1
- require 'forwardable'
1
+ require "forwardable"
2
2
 
3
3
  module HTTP
4
4
  class Headers
@@ -57,7 +57,7 @@ module HTTP
57
57
  aliases.fetch type, type.to_s
58
58
  end
59
59
 
60
- private
60
+ private
61
61
 
62
62
  # :nodoc:
63
63
  def adapters
@@ -73,4 +73,4 @@ module HTTP
73
73
  end
74
74
 
75
75
  # built-in mime types
76
- require 'http/mime_type/json'
76
+ require "http/mime_type/json"
@@ -1,5 +1,5 @@
1
- require 'forwardable'
2
- require 'singleton'
1
+ require "forwardable"
2
+ require "singleton"
3
3
 
4
4
  module HTTP
5
5
  module MimeType
@@ -1,5 +1,5 @@
1
- require 'json'
2
- require 'http/mime_type/adapter'
1
+ require "json"
2
+ require "http/mime_type/adapter"
3
3
 
4
4
  module HTTP
5
5
  module MimeType
@@ -17,7 +17,7 @@ module HTTP
17
17
  end
18
18
  end
19
19
 
20
- register_adapter 'application/json', JSON
21
- register_alias 'application/json', :json
20
+ register_adapter "application/json", JSON
21
+ register_alias "application/json", :json
22
22
  end
23
23
  end
data/lib/http/options.rb CHANGED
@@ -1,81 +1,78 @@
1
- require 'http/headers'
2
- require 'openssl'
3
- require 'socket'
1
+ require "http/headers"
2
+ require "openssl"
3
+ require "socket"
4
+ require "http/cache/null_cache"
4
5
 
5
6
  module HTTP
6
7
  class Options
7
- # How to format the response [:object, :body, :parse_body]
8
- attr_accessor :response
9
-
10
- # HTTP headers to include in the request
11
- attr_accessor :headers
12
-
13
- # Query string params to add to the url
14
- attr_accessor :params
15
-
16
- # Form data to embed in the request
17
- attr_accessor :form
18
-
19
- # JSON data to embed in the request
20
- attr_accessor :json
21
-
22
- # Explicit request body of the request
23
- attr_accessor :body
24
-
25
- # HTTP proxy to route request
26
- attr_accessor :proxy
27
-
28
- # Socket classes
29
- attr_accessor :socket_class, :ssl_socket_class
30
-
31
- # SSL context
32
- attr_accessor :ssl_context
33
-
34
- # Follow redirects
35
- attr_accessor :follow
36
-
37
- protected :response=, :headers=, :proxy=, :params=, :form=, :json=, :follow=
38
-
39
8
  @default_socket_class = TCPSocket
40
9
  @default_ssl_socket_class = OpenSSL::SSL::SSLSocket
41
10
 
11
+ @default_cache = Http::Cache::NullCache.new
12
+
42
13
  class << self
43
14
  attr_accessor :default_socket_class, :default_ssl_socket_class
15
+ attr_accessor :default_cache
44
16
 
45
17
  def new(options = {})
46
18
  return options if options.is_a?(self)
47
19
  super
48
20
  end
21
+
22
+ def defined_options
23
+ @defined_options ||= []
24
+ end
25
+
26
+ protected
27
+
28
+ def def_option(name, &interpreter)
29
+ defined_options << name.to_sym
30
+ interpreter ||= ->(v) { v }
31
+
32
+ attr_accessor name
33
+ protected :"#{name}="
34
+
35
+ define_method(:"with_#{name}") do |value|
36
+ dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) }
37
+ end
38
+ end
49
39
  end
50
40
 
51
41
  def initialize(options = {})
52
- @response = options[:response] || :auto
53
- @proxy = options[:proxy] || {}
54
- @body = options[:body]
55
- @params = options[:params]
56
- @form = options[:form]
57
- @json = options[:json]
58
- @follow = options[:follow]
59
-
60
- @headers = HTTP::Headers.coerce(options[:headers] || {})
61
-
62
- @socket_class = options[:socket_class] || self.class.default_socket_class
63
- @ssl_socket_class = options[:ssl_socket_class] || self.class.default_ssl_socket_class
64
- @ssl_context = options[:ssl_context]
42
+ defaults = {:response => :auto,
43
+ :proxy => {},
44
+ :socket_class => self.class.default_socket_class,
45
+ :ssl_socket_class => self.class.default_ssl_socket_class,
46
+ :cache => self.class.default_cache,
47
+ :headers => {}}
48
+
49
+ opts_w_defaults = defaults.merge(options)
50
+ opts_w_defaults[:headers] = HTTP::Headers.coerce(opts_w_defaults[:headers])
51
+
52
+ opts_w_defaults.each do |(opt_name, opt_val)|
53
+ self[opt_name] = opt_val
54
+ end
55
+ end
56
+
57
+ def_option :headers do |headers|
58
+ self.headers.merge(headers)
59
+ end
60
+
61
+ %w(proxy params form json body follow response socket_class ssl_socket_class ssl_context persistent).each do |method_name|
62
+ def_option method_name
65
63
  end
66
64
 
67
- def with_headers(headers)
68
- dup do |opts|
69
- opts.headers = self.headers.merge(headers)
65
+ def_option :cache do |cache_or_cache_options|
66
+ if cache_or_cache_options.respond_to? :perform
67
+ cache_or_cache_options
68
+ else
69
+ require "http/cache"
70
+ HTTP::Cache.new(cache_or_cache_options)
70
71
  end
71
72
  end
72
73
 
73
- %w(proxy params form json body follow).each do |method_name|
74
- class_eval <<-RUBY, __FILE__, __LINE__
75
- def with_#{method_name}(value)
76
- dup { |opts| opts.#{method_name} = value }
77
- end
78
- RUBY
74
+ def persistent?
75
+ !persistent.nil? && persistent != ""
79
76
  end
80
77
 
81
78
  def [](option)
@@ -97,22 +94,10 @@ module HTTP
97
94
  end
98
95
 
99
96
  def to_hash
100
- # FIXME: hardcoding these fields blows! We should have a declarative
101
- # way of specifying all the options fields, and ensure they *all*
102
- # get serialized here, rather than manually having to add them each time
103
- {
104
- :response => response,
105
- :headers => headers.to_h,
106
- :proxy => proxy,
107
- :params => params,
108
- :form => form,
109
- :json => json,
110
- :body => body,
111
- :follow => follow,
112
- :socket_class => socket_class,
113
- :ssl_socket_class => ssl_socket_class,
114
- :ssl_context => ssl_context
115
- }
97
+ hash_pairs = self.class
98
+ .defined_options
99
+ .flat_map { |opt_name| [opt_name, self[opt_name]] }
100
+ Hash[*hash_pairs]
116
101
  end
117
102
 
118
103
  def dup
@@ -121,7 +106,13 @@ module HTTP
121
106
  dupped
122
107
  end
123
108
 
124
- private
109
+ protected
110
+
111
+ def []=(option, val)
112
+ send(:"#{option}=", val)
113
+ end
114
+
115
+ private
125
116
 
126
117
  def argument_error!(message)
127
118
  fail(Error, message, caller[1..-1])
@@ -22,7 +22,7 @@ module HTTP
22
22
  follow(&block)
23
23
  end
24
24
 
25
- private
25
+ private
26
26
 
27
27
  # Reset redirector state
28
28
  def reset(request, response)
@@ -38,8 +38,8 @@ module HTTP
38
38
  fail TooManyRedirectsError if too_many_hops?
39
39
  fail EndlessRedirectError if endless_loop?
40
40
 
41
- uri = @response.headers['Location']
42
- fail StateError, 'no Location header in redirect' unless uri
41
+ uri = @response.headers["Location"]
42
+ fail StateError, "no Location header in redirect" unless uri
43
43
 
44
44
  if 303 == @response.code
45
45
  @request = @request.redirect uri, :get
data/lib/http/request.rb CHANGED
@@ -1,9 +1,11 @@
1
- require 'http/errors'
2
- require 'http/headers'
3
- require 'http/request/writer'
4
- require 'http/version'
5
- require 'base64'
6
- require 'uri'
1
+ require "http/errors"
2
+ require "http/headers"
3
+ require "http/request/caching"
4
+ require "http/request/writer"
5
+ require "http/version"
6
+ require "base64"
7
+ require "uri"
8
+ require "time"
7
9
 
8
10
  module HTTP
9
11
  class Request
@@ -66,7 +68,7 @@ module HTTP
66
68
  attr_reader :proxy, :body, :version
67
69
 
68
70
  # :nodoc:
69
- def initialize(verb, uri, headers = {}, proxy = {}, body = nil, version = '1.1') # rubocop:disable ParameterLists
71
+ def initialize(verb, uri, headers = {}, proxy = {}, body = nil, version = "1.1") # rubocop:disable ParameterLists
70
72
  @verb = verb.to_s.downcase.to_sym
71
73
  @uri = uri.is_a?(URI) ? uri : URI(uri.to_s)
72
74
  @scheme = @uri.scheme && @uri.scheme.to_s.downcase.to_sym
@@ -78,15 +80,15 @@ module HTTP
78
80
 
79
81
  @headers = HTTP::Headers.coerce(headers || {})
80
82
 
81
- @headers['Host'] ||= default_host
82
- @headers['User-Agent'] ||= USER_AGENT
83
+ @headers["Host"] ||= default_host
84
+ @headers["User-Agent"] ||= USER_AGENT
83
85
  end
84
86
 
85
87
  # Returns new Request with updated uri
86
88
  def redirect(uri, verb = @verb)
87
89
  uri = @uri.merge uri.to_s
88
90
  req = self.class.new(verb, uri, headers, proxy, body, version)
89
- req['Host'] = req.uri.host
91
+ req["Host"] = req.uri.host
90
92
  req
91
93
  end
92
94
 
@@ -109,7 +111,7 @@ module HTTP
109
111
  # Compute and add the Proxy-Authorization header
110
112
  def include_proxy_authorization_header
111
113
  digest = Base64.encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}").chomp
112
- headers['Proxy-Authorization'] = "Basic #{digest}"
114
+ headers["Proxy-Authorization"] = "Basic #{digest}"
113
115
  end
114
116
 
115
117
  # Compute HTTP request header for direct or proxy request
@@ -127,7 +129,12 @@ module HTTP
127
129
  using_proxy? ? proxy[:proxy_port] : uri.port
128
130
  end
129
131
 
130
- private
132
+ # @return [HTTP::Request::Caching]
133
+ def caching
134
+ Caching.new self
135
+ end
136
+
137
+ private
131
138
 
132
139
  def path_for_request_header
133
140
  if using_proxy?
@@ -139,7 +146,7 @@ module HTTP
139
146
 
140
147
  def uri_path_with_query
141
148
  path = uri_has_query? ? "#{uri.path}?#{uri.query}" : uri.path
142
- path.empty? ? '/' : path
149
+ path.empty? ? "/" : path
143
150
  end
144
151
 
145
152
  def uri_has_query?
@@ -0,0 +1,95 @@
1
+ require "http/cache/headers"
2
+
3
+ module HTTP
4
+ class Request
5
+ # Decorator for requests to provide convenience methods related to caching.
6
+ class Caching < DelegateClass(HTTP::Request)
7
+ INVALIDATING_METHODS = [:post, :put, :delete, :patch].freeze
8
+ CACHEABLE_METHODS = [:get, :head].freeze
9
+
10
+ # When was this request sent to the server
11
+ #
12
+ # @api public
13
+ attr_accessor :sent_at
14
+
15
+ # Inits a new instance
16
+ # @api private
17
+ def initialize(obj)
18
+ super
19
+ @requested_at = nil
20
+ @received_at = nil
21
+ end
22
+
23
+ # @return [HTTP::Request::Caching]
24
+ def caching
25
+ self
26
+ end
27
+
28
+ # @return [Boolean] true iff request demands the resources cache entry be invalidated
29
+ #
30
+ # @api public
31
+ def invalidates_cache?
32
+ INVALIDATING_METHODS.include?(verb) ||
33
+ cache_headers.no_store?
34
+ end
35
+
36
+ # @return [Boolean] true if request is cacheable
37
+ #
38
+ # @api public
39
+ def cacheable?
40
+ CACHEABLE_METHODS.include?(verb) &&
41
+ !cache_headers.no_store?
42
+ end
43
+
44
+ # @return [Boolean] true iff the cache control info of this
45
+ # request demands that the response be revalidated by the origin
46
+ # server.
47
+ #
48
+ # @api public
49
+ def skips_cache?
50
+ 0 == cache_headers.max_age ||
51
+ cache_headers.must_revalidate? ||
52
+ cache_headers.no_cache?
53
+ end
54
+
55
+ # @return [HTTP::Request::Caching] new request based on this
56
+ # one but conditional on the resource having changed since
57
+ # `cached_response`
58
+ #
59
+ # @api public
60
+ def conditional_on_changes_to(cached_response)
61
+ self.class.new HTTP::Request.new(
62
+ verb, uri, headers.merge(conditional_headers_for(cached_response)),
63
+ proxy, body, version).caching
64
+ end
65
+
66
+ # @return [HTTP::Cache::Headers] cache control helper for this request
67
+ # @api public
68
+ def cache_headers
69
+ @cache_headers ||= HTTP::Cache::Headers.new headers
70
+ end
71
+
72
+ def env
73
+ {"rack-cache.cache_key" => ->(r) { r.uri.to_s }}
74
+ end
75
+
76
+ private
77
+
78
+ # @return [Headers] conditional request headers
79
+ # @api private
80
+ def conditional_headers_for(cached_response)
81
+ headers = HTTP::Headers.new
82
+
83
+ cached_response.headers.get("Etag")
84
+ .each { |etag| headers.add("If-None-Match", etag) }
85
+
86
+ cached_response.headers.get("Last-Modified")
87
+ .each { |last_mod| headers.add("If-Modified-Since", last_mod) }
88
+
89
+ headers.add("Cache-Control", "max-age=0") if cache_headers.forces_revalidation?
90
+
91
+ headers
92
+ end
93
+ end
94
+ end
95
+ end