http 0.7.4 → 0.8.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/.rspec +0 -1
  3. data/.rubocop.yml +5 -2
  4. data/CHANGES.md +24 -7
  5. data/CONTRIBUTING.md +25 -0
  6. data/Gemfile +24 -22
  7. data/Guardfile +2 -2
  8. data/README.md +34 -4
  9. data/Rakefile +7 -7
  10. data/examples/parallel_requests_with_celluloid.rb +2 -2
  11. data/http.gemspec +12 -12
  12. data/lib/http.rb +11 -10
  13. data/lib/http/cache.rb +146 -0
  14. data/lib/http/cache/headers.rb +100 -0
  15. data/lib/http/cache/null_cache.rb +13 -0
  16. data/lib/http/chainable.rb +14 -3
  17. data/lib/http/client.rb +64 -80
  18. data/lib/http/connection.rb +139 -0
  19. data/lib/http/content_type.rb +2 -2
  20. data/lib/http/errors.rb +7 -1
  21. data/lib/http/headers.rb +21 -8
  22. data/lib/http/headers/mixin.rb +1 -1
  23. data/lib/http/mime_type.rb +2 -2
  24. data/lib/http/mime_type/adapter.rb +2 -2
  25. data/lib/http/mime_type/json.rb +4 -4
  26. data/lib/http/options.rb +65 -74
  27. data/lib/http/redirector.rb +3 -3
  28. data/lib/http/request.rb +20 -13
  29. data/lib/http/request/caching.rb +95 -0
  30. data/lib/http/request/writer.rb +5 -5
  31. data/lib/http/response.rb +15 -9
  32. data/lib/http/response/body.rb +21 -8
  33. data/lib/http/response/caching.rb +142 -0
  34. data/lib/http/response/io_body.rb +63 -0
  35. data/lib/http/response/parser.rb +1 -1
  36. data/lib/http/response/status.rb +4 -12
  37. data/lib/http/response/status/reasons.rb +53 -53
  38. data/lib/http/response/string_body.rb +53 -0
  39. data/lib/http/version.rb +1 -1
  40. data/spec/lib/http/cache/headers_spec.rb +77 -0
  41. data/spec/lib/http/cache_spec.rb +182 -0
  42. data/spec/lib/http/client_spec.rb +123 -95
  43. data/spec/lib/http/content_type_spec.rb +25 -25
  44. data/spec/lib/http/headers/mixin_spec.rb +8 -8
  45. data/spec/lib/http/headers_spec.rb +213 -173
  46. data/spec/lib/http/options/body_spec.rb +5 -5
  47. data/spec/lib/http/options/form_spec.rb +3 -3
  48. data/spec/lib/http/options/headers_spec.rb +7 -7
  49. data/spec/lib/http/options/json_spec.rb +3 -3
  50. data/spec/lib/http/options/merge_spec.rb +26 -22
  51. data/spec/lib/http/options/new_spec.rb +10 -10
  52. data/spec/lib/http/options/proxy_spec.rb +8 -8
  53. data/spec/lib/http/options_spec.rb +2 -2
  54. data/spec/lib/http/redirector_spec.rb +32 -32
  55. data/spec/lib/http/request/caching_spec.rb +133 -0
  56. data/spec/lib/http/request/writer_spec.rb +26 -26
  57. data/spec/lib/http/request_spec.rb +63 -58
  58. data/spec/lib/http/response/body_spec.rb +13 -13
  59. data/spec/lib/http/response/caching_spec.rb +201 -0
  60. data/spec/lib/http/response/io_body_spec.rb +35 -0
  61. data/spec/lib/http/response/status_spec.rb +25 -25
  62. data/spec/lib/http/response/string_body_spec.rb +35 -0
  63. data/spec/lib/http/response_spec.rb +64 -45
  64. data/spec/lib/http_spec.rb +103 -76
  65. data/spec/spec_helper.rb +10 -12
  66. data/spec/support/connection_reuse_shared.rb +100 -0
  67. data/spec/support/create_certs.rb +12 -12
  68. data/spec/support/dummy_server.rb +11 -11
  69. data/spec/support/dummy_server/servlet.rb +43 -31
  70. data/spec/support/proxy_server.rb +31 -25
  71. metadata +57 -8
  72. data/spec/support/example_server.rb +0 -30
  73. data/spec/support/example_server/servlet.rb +0 -102
@@ -0,0 +1,100 @@
1
+ require "delegate"
2
+
3
+ require "http/errors"
4
+ require "http/headers"
5
+
6
+ module HTTP
7
+ class Cache
8
+ # Convenience methods around cache control headers.
9
+ class Headers < ::SimpleDelegator
10
+ def initialize(headers)
11
+ if headers.is_a? HTTP::Headers
12
+ super headers
13
+ else
14
+ super HTTP::Headers.coerce headers
15
+ end
16
+ end
17
+
18
+ # @return [Boolean] does this message force revalidation
19
+ def forces_revalidation?
20
+ must_revalidate? || max_age == 0
21
+ end
22
+
23
+ # @return [Boolean] does the cache control include 'must-revalidate'
24
+ def must_revalidate?
25
+ matches?(/\bmust-revalidate\b/i)
26
+ end
27
+
28
+ # @return [Boolean] does the cache control include 'no-cache'
29
+ def no_cache?
30
+ matches?(/\bno-cache\b/i)
31
+ end
32
+
33
+ # @return [Boolean] does the cache control include 'no-stor'
34
+ def no_store?
35
+ matches?(/\bno-store\b/i)
36
+ end
37
+
38
+ # @return [Boolean] does the cache control include 'public'
39
+ def public?
40
+ matches?(/\bpublic\b/i)
41
+ end
42
+
43
+ # @return [Boolean] does the cache control include 'private'
44
+ def private?
45
+ matches?(/\bprivate\b/i)
46
+ end
47
+
48
+ # @return [Numeric] the max number of seconds this message is
49
+ # considered fresh.
50
+ def max_age
51
+ explicit_max_age || seconds_til_expires || Float::INFINITY
52
+ end
53
+
54
+ # @return [Boolean] is the vary header set to '*'
55
+ def vary_star?
56
+ get("Vary").any? { |v| "*" == v.strip }
57
+ end
58
+
59
+ private
60
+
61
+ # @return [Boolean] true when cache-control header matches the pattern
62
+ def matches?(pattern)
63
+ get("Cache-Control").any? { |v| v =~ pattern }
64
+ end
65
+
66
+ # @return [Numeric] number of seconds until the time in the
67
+ # expires header is reached.
68
+ #
69
+ # ---
70
+ # Some servers send a "Expire: -1" header which must be treated as expired
71
+ def seconds_til_expires
72
+ get("Expires")
73
+ .map { |e| http_date_to_ttl(e) }
74
+ .max
75
+ end
76
+
77
+ def http_date_to_ttl(t_str)
78
+ ttl = to_time_or_epoch(t_str) - Time.now
79
+
80
+ ttl < 0 ? 0 : ttl
81
+ end
82
+
83
+ # @return [Time] parses t_str at a time; if that fails returns epoch time
84
+ def to_time_or_epoch(t_str)
85
+ Time.httpdate(t_str)
86
+ rescue ArgumentError
87
+ Time.at(0)
88
+ end
89
+
90
+ # @return [Numeric] the value of the max-age component of cache control
91
+ def explicit_max_age
92
+ get("Cache-Control")
93
+ .map { |v| (/max-age=(\d+)/i).match(v) }
94
+ .compact
95
+ .map { |m| m[1].to_i }
96
+ .max
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,13 @@
1
+ module HTTP
2
+ class Cache
3
+ # NoOp cache. Always makes the request. Allows avoiding
4
+ # conditionals in the request flow.
5
+ class NullCache
6
+ # @return [Response] the result of the provided block
7
+ # @yield [request, options] so that the request can actually be made
8
+ def perform(request, options)
9
+ yield(request, options)
10
+ end
11
+ end
12
+ end
13
+ end
@@ -1,4 +1,4 @@
1
- require 'base64'
1
+ require "base64"
2
2
 
3
3
  module HTTP
4
4
  module Chainable
@@ -72,6 +72,13 @@ module HTTP
72
72
  branch(options).request verb, uri
73
73
  end
74
74
 
75
+ # Flag as persistent
76
+ # @param [String] host
77
+ # @raise [Request::Error] if Host is invalid
78
+ def persistent(host)
79
+ branch default_options.with_persistent host
80
+ end
81
+
75
82
  # Make a request through an HTTP proxy
76
83
  # @param [Array] proxy
77
84
  # @raise [Request::Error] if HTTP proxy is invalid
@@ -107,6 +114,10 @@ module HTTP
107
114
  # @see #follow
108
115
  alias_method :with_follow, :follow
109
116
 
117
+ def with_cache(cache)
118
+ branch default_options.with_cache(cache)
119
+ end
120
+
110
121
  # Make a request with the given headers
111
122
  # @param headers
112
123
  def with_headers(headers)
@@ -138,7 +149,7 @@ module HTTP
138
149
  user = opts.fetch :user
139
150
  pass = opts.fetch :pass
140
151
 
141
- auth('Basic ' << Base64.strict_encode64("#{user}:#{pass}"))
152
+ auth("Basic " << Base64.strict_encode64("#{user}:#{pass}"))
142
153
  end
143
154
 
144
155
  # Get options for HTTP
@@ -167,7 +178,7 @@ module HTTP
167
178
  end
168
179
  end
169
180
 
170
- private
181
+ private
171
182
 
172
183
  # :nodoc:
173
184
  def branch(options)
data/lib/http/client.rb CHANGED
@@ -1,23 +1,22 @@
1
- require 'cgi'
2
- require 'uri'
3
- require 'http/form_data'
4
- require 'http/options'
5
- require 'http/redirector'
1
+ require "cgi"
2
+ require "uri"
3
+ require "http/form_data"
4
+ require "http/options"
5
+ require "http/redirector"
6
6
 
7
7
  module HTTP
8
8
  # Clients make requests and receive responses
9
9
  class Client
10
10
  include Chainable
11
11
 
12
- # Input buffer size
13
- BUFFER_SIZE = 16_384
12
+ CONNECTION = "Connection".freeze
13
+ KEEP_ALIVE = "Keep-Alive".freeze
14
+ CLOSE = "close".freeze
14
15
 
15
16
  attr_reader :default_options
16
17
 
17
18
  def initialize(default_options = {})
18
19
  @default_options = HTTP::Options.new(default_options)
19
- @parser = HTTP::Response::Parser.new
20
- @socket = nil
21
20
  end
22
21
 
23
22
  # Make an HTTP request
@@ -28,6 +27,13 @@ module HTTP
28
27
  proxy = opts.proxy
29
28
  body = make_request_body(opts, headers)
30
29
 
30
+ # Tell the server to keep the conn open
31
+ if default_options.persistent?
32
+ headers[CONNECTION] = KEEP_ALIVE
33
+ else
34
+ headers[CONNECTION] = CLOSE
35
+ end
36
+
31
37
  req = HTTP::Request.new(verb, uri, headers, proxy, body)
32
38
  res = perform req, opts
33
39
 
@@ -42,66 +48,61 @@ module HTTP
42
48
 
43
49
  # Perform a single (no follow) HTTP request
44
50
  def perform(req, options)
45
- # finish previous response if client was re-used
46
- # TODO: this is pretty wrong, as socket shoud be part of response
47
- # connection, so that re-use of client will not break multiple
48
- # chunked responses
49
- finish_response
50
-
51
- uri = req.uri
52
-
53
- # TODO: keep-alive support
54
- @socket = options[:socket_class].open(req.socket_host, req.socket_port)
55
- @socket = start_tls(@socket, uri.host, options) if uri.is_a?(URI::HTTPS) && !req.using_proxy?
51
+ options.cache.perform(req, options) do |r, opts|
52
+ make_request(r, opts)
53
+ end
54
+ end
56
55
 
57
- req.stream @socket
56
+ def make_request(req, options)
57
+ verify_connection!(req.uri)
58
58
 
59
- read_headers!
59
+ @connection ||= HTTP::Connection.new(req, options)
60
+ @connection.send_request(req)
61
+ @connection.read_headers!
60
62
 
61
- body = Response::Body.new(self)
62
- res = Response.new(@parser.status_code, @parser.http_version, @parser.headers, body, uri)
63
+ res = Response.new(
64
+ @connection.parser.status_code,
65
+ @connection.parser.http_version,
66
+ @connection.parser.headers,
67
+ Response::Body.new(@connection),
68
+ req.uri
69
+ )
63
70
 
64
- finish_response if :head == req.verb
71
+ @connection.finish_response if req.verb == :head
65
72
 
66
73
  res
67
- end
68
74
 
69
- # Read a chunk of the body
70
- #
71
- # @return [String] data chunk
72
- # @return [Nil] when no more data left
73
- def readpartial(size = BUFFER_SIZE)
74
- return unless @socket
75
-
76
- begin
77
- read_more size
78
- finished = @parser.finished?
79
- rescue EOFError
80
- finished = true
75
+ # On any exception we reset the conn. This is a safety measure, to ensure
76
+ # we don't have conns in a bad state resulting in mixed requests/responses
77
+ rescue
78
+ if default_options.persistent? && @connection
79
+ @connection.close
80
+ @connection = nil
81
81
  end
82
82
 
83
- chunk = @parser.chunk
84
-
85
- finish_response if finished
86
-
87
- chunk.to_s
83
+ raise
88
84
  end
89
85
 
90
- private
86
+ private
91
87
 
92
- # Initialize TLS connection
93
- def start_tls(socket, host, options)
94
- # TODO: abstract away SSLContexts so we can use other TLS libraries
95
- context = options[:ssl_context] || OpenSSL::SSL::SSLContext.new
96
- socket = options[:ssl_socket_class].new(socket, context)
88
+ # Verify our request isn't going to be made against another URI
89
+ def verify_connection!(uri)
90
+ if default_options.persistent? && base_host(uri) != default_options.persistent
91
+ fail StateError, "Persistence is enabled for #{default_options.persistent}, but we got #{base_host(uri)}"
97
92
 
98
- socket.connect
99
-
100
- if context.verify_mode == OpenSSL::SSL::VERIFY_PEER
101
- socket.post_connection_check(host)
93
+ # We re-create the connection object because we want to let prior requests
94
+ # lazily load the body as long as possible, and this mimics prior functionality.
95
+ elsif !default_options.persistent? || (@connection && !@connection.keep_alive?)
96
+ @connection = nil
102
97
  end
98
+ end
103
99
 
104
- socket
100
+ # Strips out query/path to give us a consistent way of comparing hosts
101
+ def base_host(uri)
102
+ base = uri.dup
103
+ base.query = nil
104
+ base.path = ""
105
+ base.to_s
105
106
  end
106
107
 
107
108
  # Merges query params if needed
@@ -121,12 +122,16 @@ module HTTP
121
122
  # @param [#to_s] uri
122
123
  # @return [URI]
123
124
  def normalize_uri(uri)
124
- uri = URI uri.to_s
125
+ if default_options.persistent? && uri !~ /^http|https/
126
+ uri = URI("#{default_options.persistent}#{uri}")
127
+ else
128
+ uri = URI(uri.to_s)
129
+ end
125
130
 
126
131
  # Some proxies (seen on WEBRick) fail if URL has
127
132
  # empty path (e.g. `http://example.com`) while it's RFC-complaint:
128
133
  # http://tools.ietf.org/html/rfc1738#section-3.1
129
- uri.path = '/' if uri.path.empty?
134
+ uri.path = "/" if uri.path.empty?
130
135
 
131
136
  uri
132
137
  end
@@ -138,34 +143,13 @@ module HTTP
138
143
  opts.body
139
144
  when opts.form
140
145
  form = HTTP::FormData.create opts.form
141
- headers['Content-Type'] ||= form.content_type
142
- headers['Content-Length'] ||= form.content_length
146
+ headers["Content-Type"] ||= form.content_type
147
+ headers["Content-Length"] ||= form.content_length
143
148
  form.to_s
144
149
  when opts.json
145
- headers['Content-Type'] ||= 'application/json'
150
+ headers["Content-Type"] ||= "application/json"
146
151
  MimeType[:json].encode opts.json
147
152
  end
148
153
  end
149
-
150
- # Reads data from socket up until headers
151
- def read_headers!
152
- read_more BUFFER_SIZE until @parser.headers
153
- rescue IOError, Errno::ECONNRESET, Errno::EPIPE => ex
154
- return if ex.is_a?(EOFError) && @parser.headers
155
- raise IOError, "problem making HTTP request: #{ex}"
156
- end
157
-
158
- # Callback for when we've reached the end of a response
159
- def finish_response
160
- @socket.close if @socket && !@socket.closed?
161
- @parser.reset
162
-
163
- @socket = nil
164
- end
165
-
166
- # Feeds some more data into parser
167
- def read_more(size)
168
- @parser << @socket.readpartial(size) unless @parser.finished?
169
- end
170
154
  end
171
155
  end
@@ -0,0 +1,139 @@
1
+ require "http/response/parser"
2
+
3
+ module HTTP
4
+ # A connection to the HTTP server
5
+ class Connection
6
+ attr_reader :socket, :parser, :persistent,
7
+ :pending_request, :pending_response, :sequence_id
8
+
9
+ # Attempt to read this much data
10
+ BUFFER_SIZE = 16_384
11
+
12
+ def initialize(req, options)
13
+ @persistent = options.persistent?
14
+
15
+ @parser = Response::Parser.new
16
+ @sequence_id = 0
17
+
18
+ @socket = options[:socket_class].open(req.socket_host, req.socket_port)
19
+
20
+ start_tls(req.uri.host, options[:ssl_socket_class], options[:ssl_context]) if req.uri.is_a?(URI::HTTPS) && !req.using_proxy?
21
+ end
22
+
23
+ # Send a request to the server
24
+ def send_request(req)
25
+ if pending_request
26
+ fail StateError, "Tried to send a request while one is pending already. This cannot be called from multiple threads!"
27
+ elsif pending_request
28
+ fail StateError, "Tried to send a request while a response is pending. Make sure you've fully read the body from the request."
29
+ end
30
+
31
+ @pending_request = true
32
+ @sequence_id += 1
33
+
34
+ req.stream socket
35
+
36
+ @pending_response = true
37
+ @pending_request = nil
38
+ end
39
+
40
+ # Read a chunk of the body
41
+ #
42
+ # @return [String] data chunk
43
+ # @return [Nil] when no more data left
44
+ def readpartial(size = BUFFER_SIZE)
45
+ return unless pending_response
46
+
47
+ begin
48
+ read_more size
49
+ finished = parser.finished?
50
+ rescue EOFError
51
+ finished = true
52
+ end
53
+
54
+ chunk = parser.chunk
55
+
56
+ finish_response if finished
57
+
58
+ chunk.to_s
59
+ end
60
+
61
+ # Reads data from socket up until headers
62
+ def read_headers!
63
+ read_more BUFFER_SIZE until parser.headers
64
+ set_keep_alive
65
+
66
+ rescue IOError, Errno::ECONNRESET, Errno::EPIPE => ex
67
+ return if ex.is_a?(EOFError) && parser.headers
68
+ raise IOError, "problem making HTTP request: #{ex}"
69
+ end
70
+
71
+ # Callback for when we've reached the end of a response
72
+ def finish_response
73
+ close unless keep_alive?
74
+
75
+ parser.reset
76
+
77
+ @pending_response = nil
78
+ end
79
+
80
+ # Close the connection
81
+ def close
82
+ socket.close unless socket.closed?
83
+
84
+ @pending_response = nil
85
+ @pending_request = nil
86
+ end
87
+
88
+ # Whether we're keeping the conn alive
89
+ def keep_alive?
90
+ !!@keep_alive && !socket.closed?
91
+ end
92
+
93
+ # Store whether the connection should be kept alive.
94
+ # Once we reset the parser, we lose all of this state.
95
+ def set_keep_alive
96
+ return @keep_alive = false unless persistent
97
+
98
+ # HTTP/1.0 requires opt in for Keep Alive
99
+ if parser.http_version == "1.0"
100
+ @keep_alive = parser.headers["Connection"] == HTTP::Client::KEEP_ALIVE
101
+
102
+ # HTTP/1.1 is opt-out
103
+ elsif parser.http_version == "1.1"
104
+ @keep_alive = parser.headers["Connection"] != HTTP::Client::CLOSE
105
+
106
+ # Anything else we assume doesn't supportit
107
+ else
108
+ @keep_alive = false
109
+ end
110
+ end
111
+
112
+ private :set_keep_alive
113
+
114
+ # Feeds some more data into parser
115
+ def read_more(size)
116
+ parser << socket.readpartial(size) unless parser.finished?
117
+ end
118
+
119
+ private :read_more
120
+
121
+ # Starts the SSL connection
122
+ def start_tls(host, ssl_socket_class, ssl_context)
123
+ # TODO: abstract away SSLContexts so we can use other TLS libraries
124
+ ssl_context ||= OpenSSL::SSL::SSLContext.new
125
+ @socket = ssl_socket_class.new(socket, ssl_context)
126
+ socket.sync_close = true
127
+
128
+ socket.connect
129
+
130
+ if ssl_context.verify_mode == OpenSSL::SSL::VERIFY_PEER
131
+ socket.post_connection_check(host)
132
+ end
133
+
134
+ socket
135
+ end
136
+
137
+ private :start_tls
138
+ end
139
+ end