httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -3,7 +3,7 @@
3
3
  require "httpx/parser/http1"
4
4
 
5
5
  module HTTPX
6
- class Channel::HTTP1
6
+ class Connection::HTTP1
7
7
  include Callbacks
8
8
  include Loggable
9
9
 
@@ -39,7 +39,7 @@ module HTTPX
39
39
  @parser << data
40
40
  end
41
41
 
42
- def send(request, **)
42
+ def send(request)
43
43
  if @max_requests.positive? &&
44
44
  @requests.size >= @max_concurrent_requests
45
45
  @pending << request
@@ -79,11 +79,11 @@ module HTTPX
79
79
  return if @request.response
80
80
 
81
81
  log(level: 2) { "headers received" }
82
- headers = @options.headers_class.new(h)
83
- response = @options.response_class.new(@request,
84
- @parser.status_code,
85
- @parser.http_version.join("."),
86
- headers, @options)
82
+ headers = @request.options.headers_class.new(h)
83
+ response = @request.options.response_class.new(@request,
84
+ @parser.status_code,
85
+ @parser.http_version.join("."),
86
+ headers)
87
87
  log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
88
88
  log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
89
89
 
@@ -93,6 +93,7 @@ module HTTPX
93
93
 
94
94
  def on_trailers(h)
95
95
  return unless @request
96
+
96
97
  response = @request.response
97
98
  log(level: 2) { "trailer headers received" }
98
99
 
@@ -102,6 +103,7 @@ module HTTPX
102
103
 
103
104
  def on_data(chunk)
104
105
  return unless @request
106
+
105
107
  log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
106
108
  log(level: 2, color: :green) { "-> #{chunk.inspect}" }
107
109
  response = @request.response
@@ -111,6 +113,7 @@ module HTTPX
111
113
 
112
114
  def on_complete
113
115
  return unless @request
116
+
114
117
  log(level: 2) { "parsing complete" }
115
118
  dispatch
116
119
  end
@@ -140,9 +143,7 @@ module HTTPX
140
143
 
141
144
  def handle_error(ex)
142
145
  if @pipelining
143
- disable_pipelining
144
- emit(:reset)
145
- throw(:called)
146
+ disable
146
147
  else
147
148
  @requests.each do |request|
148
149
  emit(:error, request, ex)
@@ -158,6 +159,7 @@ module HTTPX
158
159
  when /keep\-alive/i
159
160
  keep_alive = response.headers["keep-alive"]
160
161
  return unless keep_alive
162
+
161
163
  parameters = Hash[keep_alive.split(/ *, */).map do |pair|
162
164
  pair.split(/ *= */)
163
165
  end]
@@ -166,16 +168,27 @@ module HTTPX
166
168
  keep_alive_timeout = parameters["timeout"].to_i
167
169
  emit(:timeout, keep_alive_timeout)
168
170
  end
169
- # TODO: on keep alive timeout, reset
170
- when /close/i, nil
171
- disable_pipelining
171
+ when /close/i
172
172
  @max_requests = Float::INFINITY
173
- emit(:reset)
173
+ disable
174
+ when nil
175
+ # In HTTP/1.1, it's keep alive by default
176
+ return if response.version == "1.1"
177
+
178
+ @max_requests = Float::INFINITY
179
+ disable
174
180
  end
175
181
  end
176
182
 
183
+ def disable
184
+ disable_pipelining
185
+ emit(:reset)
186
+ throw(:called)
187
+ end
188
+
177
189
  def disable_pipelining
178
190
  return if @requests.empty?
191
+
179
192
  @requests.each { |r| r.transition(:idle) }
180
193
  # server doesn't handle pipelining, and probably
181
194
  # doesn't support keep-alive. Fallback to send only
@@ -226,6 +239,7 @@ module HTTPX
226
239
 
227
240
  def join_body(request)
228
241
  return if request.empty?
242
+
229
243
  while (chunk = request.drain_body)
230
244
  log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
231
245
  log(level: 2, color: :green) { "<- #{chunk.inspect}" }
@@ -243,5 +257,5 @@ module HTTPX
243
257
  UPCASED[field] || field.to_s.split("-").map(&:capitalize).join("-")
244
258
  end
245
259
  end
246
- Channel.register "http/1.1", Channel::HTTP1
260
+ Connection.register "http/1.1", Connection::HTTP1
247
261
  end
@@ -4,7 +4,7 @@ require "io/wait"
4
4
  require "http/2"
5
5
 
6
6
  module HTTPX
7
- class Channel::HTTP2
7
+ class Connection::HTTP2
8
8
  include Callbacks
9
9
  include Loggable
10
10
 
@@ -158,6 +158,8 @@ module HTTPX
158
158
  end
159
159
 
160
160
  def join_body(stream, request)
161
+ return if request.empty?
162
+
161
163
  chunk = @drains.delete(request) || request.drain_body
162
164
  while chunk
163
165
  next_chunk = request.drain_body
@@ -181,8 +183,8 @@ module HTTPX
181
183
  h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n")
182
184
  end
183
185
  _, status = h.shift
184
- headers = @options.headers_class.new(h)
185
- response = @options.response_class.new(request, status, "2.0", headers, @options)
186
+ headers = request.options.headers_class.new(h)
187
+ response = request.options.response_class.new(request, status, "2.0", headers)
186
188
  request.response = response
187
189
  @streams[request] = stream
188
190
  end
@@ -195,6 +197,7 @@ module HTTPX
195
197
 
196
198
  def on_stream_close(stream, request, error)
197
199
  return handle(request, stream) if request.expects?
200
+
198
201
  if error
199
202
  ex = Error.new(stream.id, error)
200
203
  ex.set_backtrace(caller)
@@ -230,9 +233,12 @@ module HTTPX
230
233
  if error
231
234
  ex = Error.new(0, error)
232
235
  ex.set_backtrace(caller)
233
- emit(:error, request, ex)
236
+ @streams.each_key do |request|
237
+ emit(:error, request, ex)
238
+ end
234
239
  end
235
240
  return unless @connection.state == :closed && @connection.active_stream_count.zero?
241
+
236
242
  emit(:close)
237
243
  end
238
244
 
@@ -269,7 +275,7 @@ module HTTPX
269
275
  end
270
276
 
271
277
  def on_promise(stream)
272
- emit(:promise, self, stream)
278
+ emit(:promise, @streams.key(stream.parent), stream)
273
279
  end
274
280
 
275
281
  def respond_to_missing?(meth, *args)
@@ -284,5 +290,5 @@ module HTTPX
284
290
  end
285
291
  end
286
292
  end
287
- Channel.register "h2", Channel::HTTP2
293
+ Connection.register "h2", Connection::HTTP2
288
294
  end
@@ -20,6 +20,8 @@ module HTTPX
20
20
  end
21
21
  end
22
22
 
23
+ TotalTimeoutError = Class.new(TimeoutError)
24
+
23
25
  ConnectTimeoutError = Class.new(TimeoutError)
24
26
 
25
27
  ResolveError = Class.new(Error)
@@ -7,6 +7,7 @@ module HTTPX
7
7
  class << self
8
8
  def new(headers = nil)
9
9
  return headers if headers.is_a?(self)
10
+
10
11
  super
11
12
  end
12
13
  end
@@ -14,6 +15,7 @@ module HTTPX
14
15
  def initialize(headers = nil)
15
16
  @headers = {}
16
17
  return unless headers
18
+
17
19
  headers.each do |field, value|
18
20
  array_value(value).each do |v|
19
21
  add(downcased(field), v)
@@ -44,7 +46,6 @@ module HTTPX
44
46
  # ignore what the +other+ headers has. Otherwise, set
45
47
  #
46
48
  def merge(other)
47
- # TODO: deep-copy
48
49
  headers = dup
49
50
  other.each do |field, value|
50
51
  headers[field] = value
@@ -64,6 +65,7 @@ module HTTPX
64
65
  #
65
66
  def []=(field, value)
66
67
  return unless value
68
+
67
69
  @headers[downcased(field)] = array_value(value)
68
70
  end
69
71
 
@@ -94,6 +96,7 @@ module HTTPX
94
96
  #
95
97
  def each
96
98
  return enum_for(__method__) { @headers.size } unless block_given?
99
+
97
100
  @headers.each do |field, value|
98
101
  yield(field, value.join(", ")) unless value.empty?
99
102
  end
@@ -31,6 +31,7 @@ module HTTPX
31
31
  def verify_hostname(host)
32
32
  return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
33
33
  return false if @io.peer_cert.nil?
34
+
34
35
  OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
35
36
  end
36
37
 
@@ -54,6 +55,7 @@ module HTTPX
54
55
  end
55
56
  return if @state == :negotiated ||
56
57
  @state != :connected
58
+
57
59
  unless @io.is_a?(OpenSSL::SSL::SSLSocket)
58
60
  @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
59
61
  @io.hostname = @hostname
@@ -112,8 +114,10 @@ module HTTPX
112
114
 
113
115
  def log_transition_state(nextstate)
114
116
  return super unless nextstate == :negotiated
117
+
115
118
  server_cert = @io.peer_cert
116
- "SSL connection using #{@io.ssl_version} / #{@io.cipher.first}\n" \
119
+ "#{super}\n\n" \
120
+ "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
117
121
  "ALPN, server accepted to use #{protocol}\n" \
118
122
  "Server certificate:\n" \
119
123
  " subject: #{server_cert.subject}\n" \
@@ -13,28 +13,29 @@ module HTTPX
13
13
 
14
14
  alias_method :host, :ip
15
15
 
16
- def initialize(uri, addresses, options)
16
+ def initialize(origin, addresses, options)
17
17
  @state = :idle
18
- @hostname = uri.host
18
+ @hostname = origin.host
19
19
  @addresses = addresses
20
- @ip_index = @addresses.size - 1
21
20
  @options = Options.new(options)
22
21
  @fallback_protocol = @options.fallback_protocol
23
- @port = uri.port
22
+ @port = origin.port
24
23
  if @options.io
25
24
  @io = case @options.io
26
25
  when Hash
27
- @ip = @addresses[@ip_index]
28
- @options.io[@ip] || @options.io["#{@ip}:#{@port}"]
26
+ @options.io[origin.authority]
29
27
  else
30
- @ip = @hostname
31
28
  @options.io
32
29
  end
30
+ _, _, _, @ip = @io.addr
31
+ @addresses ||= [@ip]
32
+ @ip_index = @addresses.size - 1
33
33
  unless @io.nil?
34
34
  @keep_open = true
35
35
  @state = :connected
36
36
  end
37
37
  else
38
+ @ip_index = @addresses.size - 1
38
39
  @ip = @addresses[@ip_index]
39
40
  end
40
41
  @io ||= build_socket
@@ -54,6 +55,7 @@ module HTTPX
54
55
 
55
56
  def connect
56
57
  return unless closed?
58
+
57
59
  begin
58
60
  if @io.closed?
59
61
  transition(:idle)
@@ -65,6 +67,7 @@ module HTTPX
65
67
  transition(:connected)
66
68
  rescue Errno::EHOSTUNREACH => e
67
69
  raise e if @ip_index <= 0
70
+
68
71
  @ip_index -= 1
69
72
  retry
70
73
  rescue Errno::EINPROGRESS,
@@ -96,6 +99,7 @@ module HTTPX
96
99
  ret = @io.read_nonblock(size, buffer, exception: false)
97
100
  return 0 if ret == :wait_readable
98
101
  return if ret.nil?
102
+
99
103
  buffer.bytesize
100
104
  end
101
105
 
@@ -103,6 +107,7 @@ module HTTPX
103
107
  siz = @io.write_nonblock(buffer, exception: false)
104
108
  return 0 if siz == :wait_writable
105
109
  return if siz.nil?
110
+
106
111
  buffer.slice!(0, siz)
107
112
  siz
108
113
  end
@@ -110,6 +115,7 @@ module HTTPX
110
115
 
111
116
  def close
112
117
  return if @keep_open || closed?
118
+
113
119
  begin
114
120
  @io.close
115
121
  ensure
@@ -48,6 +48,7 @@ module HTTPX
48
48
  ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
49
49
  return 0 if ret == :wait_readable
50
50
  return if ret.nil?
51
+
51
52
  buffer.bytesize
52
53
  rescue IOError
53
54
  end
@@ -36,6 +36,7 @@ module HTTPX
36
36
 
37
37
  def connect
38
38
  return unless closed?
39
+
39
40
  begin
40
41
  if @io.closed?
41
42
  transition(:idle)
@@ -3,22 +3,47 @@
3
3
  module HTTPX
4
4
  module Loggable
5
5
  COLORS = {
6
- black: 30,
7
- red: 31,
8
- green: 32,
9
- yellow: 33,
10
- blue: 34,
6
+ black: 30,
7
+ red: 31,
8
+ green: 32,
9
+ yellow: 33,
10
+ blue: 34,
11
11
  magenta: 35,
12
- cyan: 36,
13
- white: 37,
12
+ cyan: 36,
13
+ white: 37,
14
14
  }.freeze
15
15
 
16
16
  def log(level: @options.debug_level, label: "", color: nil, &msg)
17
17
  return unless @options.debug
18
18
  return unless @options.debug_level >= level
19
+
20
+ debug_stream = @options.debug
21
+
19
22
  message = (+label << msg.call << "\n")
20
- message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && @options.debug.isatty
21
- @options.debug << message
23
+ message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
24
+ debug_stream << message
25
+ end
26
+
27
+ if !Exception.instance_methods.include?(:full_message)
28
+
29
+ def log_exception(ex, level: @options.debug_level, label: "", color: nil)
30
+ return unless @options.debug
31
+ return unless @options.debug_level >= level
32
+
33
+ message = +"#{ex.message} (#{ex.class})"
34
+ message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
35
+ log(level: level, label: label, color: color) { message }
36
+ end
37
+
38
+ else
39
+
40
+ def log_exception(ex, level: @options.debug_level, label: "", color: nil)
41
+ return unless @options.debug
42
+ return unless @options.debug_level >= level
43
+
44
+ log(level: level, label: label, color: color) { ex.full_message }
45
+ end
46
+
22
47
  end
23
48
  end
24
49
  end
@@ -16,6 +16,7 @@ module HTTPX
16
16
  # let enhanced options go through
17
17
  return options if self == Options && options.class > self
18
18
  return options if options.is_a?(self)
19
+
19
20
  super
20
21
  end
21
22
 
@@ -31,31 +32,35 @@ module HTTPX
31
32
  protected :"#{name}="
32
33
 
33
34
  define_method(:"with_#{name}") do |value|
34
- dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) }
35
+ other = dup
36
+ other.send(:"#{name}=", other.instance_exec(value, &interpreter))
37
+ other
35
38
  end
36
39
  end
37
40
  end
38
41
 
39
42
  def initialize(options = {})
40
43
  defaults = {
41
- :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
42
- :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
43
- :ssl => {},
44
- :http2_settings => { settings_enable_push: 0 },
45
- :fallback_protocol => "http/1.1",
46
- :timeout => Timeout.new,
47
- :headers => {},
48
- :max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
49
- :window_size => WINDOW_SIZE,
50
- :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
51
- :request_class => Class.new(Request),
52
- :response_class => Class.new(Response),
53
- :headers_class => Class.new(Headers),
54
- :request_body_class => Class.new(Request::Body),
55
- :response_body_class => Class.new(Response::Body),
56
- :transport => nil,
57
- :transport_options => nil,
58
- :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
44
+ :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
45
+ :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
46
+ :ssl => {},
47
+ :http2_settings => { settings_enable_push: 0 },
48
+ :fallback_protocol => "http/1.1",
49
+ :timeout => Timeout.new,
50
+ :headers => {},
51
+ :max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
52
+ :window_size => WINDOW_SIZE,
53
+ :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
54
+ :request_class => Class.new(Request),
55
+ :response_class => Class.new(Response),
56
+ :headers_class => Class.new(Headers),
57
+ :request_body_class => Class.new(Request::Body),
58
+ :response_body_class => Class.new(Response::Body),
59
+ :connection_class => Class.new(Connection),
60
+ :transport => nil,
61
+ :transport_options => nil,
62
+ :persistent => false,
63
+ :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
59
64
  }
60
65
 
61
66
  defaults.merge!(options)
@@ -74,6 +79,7 @@ module HTTPX
74
79
  def_option(:max_concurrent_requests) do |num|
75
80
  max = Integer(num)
76
81
  raise Error, ":max_concurrent_requests must be positive" unless max.positive?
82
+
77
83
  self.max_concurrent_requests = max
78
84
  end
79
85
 
@@ -88,18 +94,27 @@ module HTTPX
88
94
  def_option(:transport) do |tr|
89
95
  transport = tr.to_s
90
96
  raise Error, "#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
97
+
91
98
  self.transport = transport
92
99
  end
93
100
 
94
101
  %w[
95
102
  params form json body
96
103
  follow ssl http2_settings
97
- request_class response_class headers_class request_body_class response_body_class
104
+ request_class response_class headers_class request_body_class response_body_class connection_class
98
105
  io fallback_protocol debug debug_level transport_options resolver_class resolver_options
106
+ persistent
99
107
  ].each do |method_name|
100
108
  def_option(method_name)
101
109
  end
102
110
 
111
+ def ==(other)
112
+ ivars = instance_variables | other.instance_variables
113
+ ivars.all? do |ivar|
114
+ instance_variable_get(ivar) == other.instance_variable_get(ivar)
115
+ end
116
+ end
117
+
103
118
  def merge(other)
104
119
  h1 = to_hash
105
120
  h2 = other.to_hash
@@ -123,17 +138,28 @@ module HTTPX
123
138
  Hash[*hash_pairs]
124
139
  end
125
140
 
126
- def dup
127
- dupped = super
128
- dupped.headers = headers.dup
129
- dupped.ssl = ssl.dup
130
- dupped.request_class = request_class.dup
131
- dupped.response_class = response_class.dup
132
- dupped.headers_class = headers_class.dup
133
- dupped.request_body_class = request_body_class.dup
134
- dupped.response_body_class = response_body_class.dup
135
- yield(dupped) if block_given?
136
- dupped
141
+ def initialize_dup(other)
142
+ self.headers = other.headers.dup
143
+ self.ssl = other.ssl.dup
144
+ self.request_class = other.request_class.dup
145
+ self.response_class = other.response_class.dup
146
+ self.headers_class = other.headers_class.dup
147
+ self.request_body_class = other.request_body_class.dup
148
+ self.response_body_class = other.response_body_class.dup
149
+ self.connection_class = other.connection_class.dup
150
+ end
151
+
152
+ def freeze
153
+ super
154
+
155
+ headers.freeze
156
+ ssl.freeze
157
+ request_class.freeze
158
+ response_class.freeze
159
+ headers_class.freeze
160
+ request_body_class.freeze
161
+ response_body_class.freeze
162
+ connection_class.freeze
137
163
  end
138
164
 
139
165
  protected