httpx 0.6.7 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +7 -5
  3. data/doc/release_notes/0_0_1.md +7 -0
  4. data/doc/release_notes/0_0_2.md +9 -0
  5. data/doc/release_notes/0_0_3.md +9 -0
  6. data/doc/release_notes/0_0_4.md +7 -0
  7. data/doc/release_notes/0_0_5.md +5 -0
  8. data/doc/release_notes/0_1_0.md +9 -0
  9. data/doc/release_notes/0_2_0.md +5 -0
  10. data/doc/release_notes/0_2_1.md +16 -0
  11. data/doc/release_notes/0_3_0.md +12 -0
  12. data/doc/release_notes/0_3_1.md +6 -0
  13. data/doc/release_notes/0_4_0.md +51 -0
  14. data/doc/release_notes/0_4_1.md +3 -0
  15. data/doc/release_notes/0_5_0.md +15 -0
  16. data/doc/release_notes/0_5_1.md +14 -0
  17. data/doc/release_notes/0_6_0.md +5 -0
  18. data/doc/release_notes/0_6_1.md +6 -0
  19. data/doc/release_notes/0_6_2.md +6 -0
  20. data/doc/release_notes/0_6_3.md +13 -0
  21. data/doc/release_notes/0_6_4.md +21 -0
  22. data/doc/release_notes/0_6_5.md +22 -0
  23. data/doc/release_notes/0_6_6.md +19 -0
  24. data/doc/release_notes/0_6_7.md +5 -0
  25. data/doc/release_notes/0_7_0.md +46 -0
  26. data/doc/release_notes/0_8_0.md +27 -0
  27. data/doc/release_notes/0_8_1.md +8 -0
  28. data/doc/release_notes/0_8_2.md +7 -0
  29. data/doc/release_notes/0_9_0.md +38 -0
  30. data/lib/httpx/adapters/faraday.rb +2 -2
  31. data/lib/httpx/altsvc.rb +18 -2
  32. data/lib/httpx/chainable.rb +27 -9
  33. data/lib/httpx/connection.rb +215 -65
  34. data/lib/httpx/connection/http1.rb +54 -18
  35. data/lib/httpx/connection/http2.rb +100 -37
  36. data/lib/httpx/extensions.rb +2 -2
  37. data/lib/httpx/headers.rb +2 -2
  38. data/lib/httpx/io/ssl.rb +11 -3
  39. data/lib/httpx/io/tcp.rb +12 -2
  40. data/lib/httpx/loggable.rb +6 -6
  41. data/lib/httpx/options.rb +43 -28
  42. data/lib/httpx/plugins/authentication.rb +1 -1
  43. data/lib/httpx/plugins/compression.rb +28 -8
  44. data/lib/httpx/plugins/compression/gzip.rb +22 -12
  45. data/lib/httpx/plugins/cookies.rb +12 -8
  46. data/lib/httpx/plugins/digest_authentication.rb +2 -0
  47. data/lib/httpx/plugins/expect.rb +79 -0
  48. data/lib/httpx/plugins/follow_redirects.rb +1 -2
  49. data/lib/httpx/plugins/h2c.rb +0 -1
  50. data/lib/httpx/plugins/proxy.rb +23 -20
  51. data/lib/httpx/plugins/proxy/http.rb +9 -6
  52. data/lib/httpx/plugins/proxy/socks4.rb +1 -1
  53. data/lib/httpx/plugins/proxy/socks5.rb +5 -1
  54. data/lib/httpx/plugins/proxy/ssh.rb +0 -4
  55. data/lib/httpx/plugins/push_promise.rb +2 -2
  56. data/lib/httpx/plugins/retries.rb +32 -29
  57. data/lib/httpx/pool.rb +15 -10
  58. data/lib/httpx/registry.rb +2 -1
  59. data/lib/httpx/request.rb +8 -6
  60. data/lib/httpx/resolver.rb +7 -8
  61. data/lib/httpx/resolver/https.rb +15 -3
  62. data/lib/httpx/resolver/native.rb +22 -32
  63. data/lib/httpx/resolver/options.rb +2 -2
  64. data/lib/httpx/resolver/resolver_mixin.rb +1 -1
  65. data/lib/httpx/response.rb +17 -3
  66. data/lib/httpx/selector.rb +96 -95
  67. data/lib/httpx/session.rb +33 -34
  68. data/lib/httpx/timeout.rb +7 -1
  69. data/lib/httpx/version.rb +1 -1
  70. metadata +77 -20
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "digest"
4
+
3
5
  module HTTPX
4
6
  module Plugins
5
7
  #
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
7
+ #
8
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Expect#expect
9
+ #
10
+ module Expect
11
+ EXPECT_TIMEOUT = 2
12
+
13
+ def self.extra_options(options)
14
+ Class.new(options.class) do
15
+ def_option(:expect_timeout) do |seconds|
16
+ seconds = Integer(seconds)
17
+ raise Error, ":expect_timeout must be positive" unless seconds.positive?
18
+
19
+ seconds
20
+ end
21
+
22
+ def_option(:expect_threshold_size) do |bytes|
23
+ bytes = Integer(bytes)
24
+ raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
25
+
26
+ bytes
27
+ end
28
+ end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
29
+ end
30
+
31
+ module RequestBodyMethods
32
+ def initialize(*, options)
33
+ super
34
+ return if @body.nil?
35
+
36
+ if (threshold = options.expect_threshold_size)
37
+ unless unbounded_body?
38
+ return if @body.bytesize < threshold
39
+ end
40
+ end
41
+
42
+ @headers["expect"] = "100-continue"
43
+ end
44
+ end
45
+
46
+ module ConnectionMethods
47
+ def send(request)
48
+ request.once(:expects) do
49
+ @timers.after(@options.expect_timeout) do
50
+ if request.state == :expects && !request.expects?
51
+ request.headers.delete("expect")
52
+ handle(request)
53
+ end
54
+ end
55
+ end
56
+ super
57
+ end
58
+ end
59
+
60
+ module InstanceMethods
61
+ def fetch_response(request, connections, options)
62
+ response = @responses.delete(request)
63
+ return unless response
64
+
65
+ if response.status == 417 && request.headers.key?("expect")
66
+ request.headers.delete("expect")
67
+ request.transition(:idle)
68
+ connection = find_connection(request, connections, options)
69
+ connection.send(request)
70
+ return
71
+ end
72
+
73
+ response
74
+ end
75
+ end
76
+ end
77
+ register_plugin :expect, Expect
78
+ end
79
+ end
@@ -21,7 +21,7 @@ module HTTPX
21
21
  Class.new(options.class) do
22
22
  def_option(:max_redirects) do |num|
23
23
  num = Integer(num)
24
- raise Error, ":max_redirects must be positive" unless num.positive?
24
+ raise Error, ":max_redirects must be positive" if num.negative?
25
25
 
26
26
  num
27
27
  end
@@ -61,7 +61,6 @@ module HTTPX
61
61
 
62
62
  connection = find_connection(retry_request, connections, options)
63
63
  connection.send(retry_request)
64
- set_request_timeout(connection, retry_request, options)
65
64
  nil
66
65
  end
67
66
 
@@ -42,7 +42,6 @@ module HTTPX
42
42
  connection = find_connection(request, connections, options)
43
43
  connections << connection unless connections.include?(connection)
44
44
  connection.upgrade(request, response)
45
- set_request_timeout(connection, request, options)
46
45
  end
47
46
  response
48
47
  end
@@ -68,17 +68,17 @@ module HTTPX
68
68
  def extra_options(options)
69
69
  Class.new(options.class) do
70
70
  def_option(:proxy) do |pr|
71
- Hash[pr]
71
+ if pr.is_a?(Parameters)
72
+ pr
73
+ else
74
+ Hash[pr]
75
+ end
72
76
  end
73
77
  end.new(options)
74
78
  end
75
79
  end
76
80
 
77
81
  module InstanceMethods
78
- def with_proxy(*args)
79
- branch(default_options.with_proxy(*args))
80
- end
81
-
82
82
  private
83
83
 
84
84
  def proxy_uris(uri, options)
@@ -104,7 +104,7 @@ module HTTPX
104
104
  connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
105
105
  unless connections.nil? || connections.include?(connection)
106
106
  connections << connection
107
- set_connection_callbacks(connection, options)
107
+ set_connection_callbacks(connection, connections, options)
108
108
  end
109
109
  connection
110
110
  end
@@ -129,13 +129,12 @@ module HTTPX
129
129
  connection = find_connection(request, connections, options)
130
130
  connections << connection unless connections.include?(connection)
131
131
  connection.send(request)
132
- set_request_timeout(connection, request, options)
133
132
  return
134
133
  end
135
134
  response
136
135
  end
137
136
 
138
- def build_altsvc_connection(_, _, _, _, options)
137
+ def build_altsvc_connection(_, _, _, _, _, options)
139
138
  return if options.proxy
140
139
 
141
140
  super
@@ -180,20 +179,9 @@ module HTTPX
180
179
  super || @state == :connecting || @state == :connected
181
180
  end
182
181
 
183
- def to_io
184
- return super unless @options.proxy
185
-
186
- case @state
187
- when :idle
188
- transition(:connecting)
189
- when :connected
190
- transition(:open)
191
- end
192
- @io.to_io
193
- end
194
-
195
182
  def call
196
183
  super
184
+
197
185
  return unless @options.proxy
198
186
 
199
187
  case @state
@@ -211,11 +199,26 @@ module HTTPX
211
199
  emit(:close)
212
200
  end
213
201
 
202
+ private
203
+
204
+ def connect
205
+ return super unless @options.proxy
206
+
207
+ case @state
208
+ when :idle
209
+ transition(:connecting)
210
+ when :connected
211
+ transition(:open)
212
+ end
213
+ end
214
+
214
215
  def transition(nextstate)
215
216
  return super unless @options.proxy
216
217
 
217
218
  case nextstate
218
219
  when :closing
220
+ # this is a hack so that we can use the super method
221
+ # and it'll thing that the current state is open
219
222
  @state = :open if @state == :connecting
220
223
  end
221
224
  super
@@ -7,6 +7,10 @@ module HTTPX
7
7
  module Proxy
8
8
  module HTTP
9
9
  module ConnectionMethods
10
+ def connecting?
11
+ super || @state == :connecting || @state == :connected
12
+ end
13
+
10
14
  private
11
15
 
12
16
  def transition(nextstate)
@@ -34,7 +38,6 @@ module HTTPX
34
38
  when :idle
35
39
  @parser = ProxyParser.new(@write_buffer, @options)
36
40
  set_parser_callbacks(@parser)
37
- @parser.on(:close) { transition(:closing) }
38
41
  end
39
42
  end
40
43
  super
@@ -48,7 +51,7 @@ module HTTPX
48
51
  #
49
52
  if req.uri.scheme == "https"
50
53
  connect_request = ConnectRequest.new(req.uri, @options)
51
-
54
+ @inflight += 1
52
55
  parser.send(connect_request)
53
56
  else
54
57
  transition(:connected)
@@ -56,6 +59,7 @@ module HTTPX
56
59
  end
57
60
 
58
61
  def __http_on_connect(_, response)
62
+ @inflight -= 1
59
63
  if response.status == 200
60
64
  req = @pending.first
61
65
  request_uri = req.uri
@@ -67,6 +71,7 @@ module HTTPX
67
71
  while (req = pending.shift)
68
72
  req.emit(:response, response)
69
73
  end
74
+ reset
70
75
  end
71
76
  end
72
77
  end
@@ -91,15 +96,13 @@ module HTTPX
91
96
  def headline_uri(request)
92
97
  return super unless request.verb == :connect
93
98
 
94
- uri = request.uri
95
- tunnel = "#{uri.hostname}:#{uri.port}"
99
+ tunnel = request.path
96
100
  log { "establishing HTTP proxy tunnel to #{tunnel}" }
97
101
  tunnel
98
102
  end
99
103
 
100
104
  def empty?
101
- @requests.reject { |r| r.verb == :connect }.empty? ||
102
- @requests.all? { |request| !request.response.nil? }
105
+ @requests.reject { |r| r.verb == :connect }.empty? || @requests.all? { |request| !request.response.nil? }
103
106
  end
104
107
  end
105
108
 
@@ -39,7 +39,7 @@ module HTTPX
39
39
 
40
40
  @parser = nil
41
41
  end
42
- log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
42
+ log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
43
43
  super
44
44
  end
45
45
 
@@ -31,6 +31,10 @@ module HTTPX
31
31
  end
32
32
  end
33
33
 
34
+ def connecting?
35
+ super || @state == :authenticating || @state == :negotiating
36
+ end
37
+
34
38
  private
35
39
 
36
40
  def transition(nextstate)
@@ -60,7 +64,7 @@ module HTTPX
60
64
 
61
65
  @parser = nil
62
66
  end
63
- log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
67
+ log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
64
68
  super
65
69
  end
66
70
 
@@ -19,10 +19,6 @@ module HTTPX
19
19
  end
20
20
 
21
21
  module InstanceMethods
22
- def with_proxy(*args)
23
- branch(default_options.with_proxy(*args))
24
- end
25
-
26
22
  private
27
23
 
28
24
  def send_requests(*requests, options)
@@ -43,9 +43,9 @@ module HTTPX
43
43
  end
44
44
 
45
45
  def __on_promise_request(parser, stream, h)
46
- log(level: 1, label: "#{stream.id}: ") do
46
+ log(level: 1) do
47
47
  # :nocov:
48
- h.map { |k, v| "-> PROMISE HEADER: #{k}: #{v}" }.join("\n")
48
+ h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
49
49
  # :nocov:
50
50
  end
51
51
  headers = @options.headers_class.new(h)
@@ -28,10 +28,10 @@ module HTTPX
28
28
  # number of seconds after which one can retry the request
29
29
  def_option(:retry_after) do |num|
30
30
  # return early if callable
31
- return num if num.respond_to?(:call)
32
-
33
- num = Integer(num)
34
- raise Error, ":retry_after must be positive" unless num.positive?
31
+ unless num.respond_to?(:call)
32
+ num = Integer(num)
33
+ raise Error, ":retry_after must be positive" unless num.positive?
34
+ end
35
35
 
36
36
  num
37
37
  end
@@ -46,7 +46,7 @@ module HTTPX
46
46
  def_option(:retry_change_requests)
47
47
 
48
48
  def_option(:retry_on) do |callback|
49
- raise ":retry_on must be called with the response" unless callback.respond_to?(:call) && callback.method(:call).arity == 1
49
+ raise ":retry_on must be called with the response" unless callback.respond_to?(:call)
50
50
 
51
51
  callback
52
52
  end
@@ -63,18 +63,38 @@ module HTTPX
63
63
  def fetch_response(request, connections, options)
64
64
  response = super
65
65
 
66
- retry_on = options.retry_on
67
-
68
- if response.is_a?(ErrorResponse) &&
66
+ if response &&
69
67
  request.retries.positive? &&
70
68
  __repeatable_request?(request, options) &&
71
- __retryable_error?(response.error) &&
72
- (!retry_on || retry_on.call(response))
69
+ (
70
+ # rubocop:disable Style/MultilineTernaryOperator
71
+ options.retry_on ?
72
+ options.retry_on.call(response) :
73
+ (
74
+ response.is_a?(ErrorResponse) && __retryable_error?(response.error)
75
+ )
76
+ # rubocop:enable Style/MultilineTernaryOperator
77
+ )
78
+
73
79
  request.retries -= 1
74
80
  log { "failed to get response, #{request.retries} tries to go..." }
75
81
  request.transition(:idle)
76
- connection = find_connection(request, connections, options)
77
- __retry_request(connection, request, options)
82
+
83
+ retry_after = options.retry_after
84
+ if retry_after
85
+ retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
86
+
87
+ log { "retrying after #{retry_after} secs..." }
88
+ pool.after(retry_after) do
89
+ log { "retrying!!" }
90
+ connection = find_connection(request, connections, options)
91
+ connection.send(request)
92
+ end
93
+ else
94
+ connection = find_connection(request, connections, options)
95
+ connection.send(request)
96
+ end
97
+
78
98
  return
79
99
  end
80
100
  response
@@ -87,23 +107,6 @@ module HTTPX
87
107
  def __retryable_error?(ex)
88
108
  RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
89
109
  end
90
-
91
- def __retry_request(connection, request, options)
92
- retry_after = options.retry_after
93
- unless retry_after
94
- connection.send(request)
95
- set_request_timeout(connection, request, options)
96
- return
97
- end
98
-
99
- retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
100
- log { "retrying after #{retry_after} secs..." }
101
- pool.after(retry_after) do
102
- log { "retrying!!" }
103
- connection.send(request)
104
- set_request_timeout(connection, request, options)
105
- end
106
- end
107
110
  end
108
111
 
109
112
  module RequestMethods
@@ -14,7 +14,7 @@ module HTTPX
14
14
 
15
15
  def initialize
16
16
  @resolvers = {}
17
- @_resolver_monitors = {}
17
+ @_resolver_ios = {}
18
18
  @timers = Timers::Group.new
19
19
  @selector = Selector.new
20
20
  @connections = []
@@ -27,12 +27,19 @@ module HTTPX
27
27
 
28
28
  def next_tick
29
29
  catch(:jump_tick) do
30
- @selector.select(next_timeout || @timers.wait_interval) do |monitor|
31
- monitor.io.call
32
- monitor.interests = monitor.io.interests
30
+ timeout = [next_timeout, @timers.wait_interval].compact.min
31
+ if timeout && timeout.negative?
32
+ @timers.fire
33
+ throw(:jump_tick)
33
34
  end
35
+
36
+ @selector.select(timeout, &:call)
37
+
34
38
  @timers.fire
35
39
  end
40
+ rescue Interrupt
41
+ @connections.each(&:reset)
42
+ raise
36
43
  rescue StandardError => e
37
44
  @connections.each do |connection|
38
45
  connection.emit(:error, e)
@@ -51,6 +58,7 @@ module HTTPX
51
58
 
52
59
  def init_connection(connection, _options)
53
60
  resolve_connection(connection)
61
+ connection.timers = @timers
54
62
  connection.on(:open) do
55
63
  @connected_connections += 1
56
64
  end
@@ -79,7 +87,7 @@ module HTTPX
79
87
  resolver << connection
80
88
  return if resolver.empty?
81
89
 
82
- @_resolver_monitors[resolver] ||= @selector.register(resolver, :w)
90
+ @_resolver_ios[resolver] ||= @selector.register(resolver)
83
91
  end
84
92
 
85
93
  def on_resolver_connection(connection)
@@ -111,8 +119,7 @@ module HTTPX
111
119
  @resolvers.delete(resolver_type)
112
120
 
113
121
  @selector.deregister(resolver)
114
- monitor = @_resolver_monitors.delete(resolver)
115
- monitor.close if monitor
122
+ @_resolver_ios.delete(resolver)
116
123
  resolver.close unless resolver.closed?
117
124
  end
118
125
 
@@ -121,10 +128,8 @@ module HTTPX
121
128
  # if open, an IO was passed upstream, therefore
122
129
  # consider it connected already.
123
130
  @connected_connections += 1
124
- @selector.register(connection, :rw)
125
- else
126
- @selector.register(connection, :w)
127
131
  end
132
+ @selector.register(connection)
128
133
  connection.on(:close) do
129
134
  unregister_connection(connection)
130
135
  end