httpx 0.10.0 → 0.11.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -3
  3. data/doc/release_notes/0_10_1.md +37 -0
  4. data/doc/release_notes/0_10_2.md +5 -0
  5. data/doc/release_notes/0_11_0.md +76 -0
  6. data/doc/release_notes/0_11_1.md +1 -0
  7. data/doc/release_notes/0_11_2.md +5 -0
  8. data/lib/httpx/adapters/datadog.rb +205 -0
  9. data/lib/httpx/adapters/faraday.rb +0 -2
  10. data/lib/httpx/adapters/webmock.rb +123 -0
  11. data/lib/httpx/chainable.rb +8 -7
  12. data/lib/httpx/connection.rb +4 -15
  13. data/lib/httpx/connection/http1.rb +14 -1
  14. data/lib/httpx/connection/http2.rb +15 -16
  15. data/lib/httpx/domain_name.rb +1 -3
  16. data/lib/httpx/errors.rb +3 -1
  17. data/lib/httpx/headers.rb +1 -0
  18. data/lib/httpx/io/ssl.rb +4 -8
  19. data/lib/httpx/io/udp.rb +4 -3
  20. data/lib/httpx/plugins/compression.rb +1 -1
  21. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  22. data/lib/httpx/plugins/expect.rb +33 -8
  23. data/lib/httpx/plugins/multipart.rb +42 -23
  24. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  25. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  26. data/lib/httpx/plugins/multipart/part.rb +34 -0
  27. data/lib/httpx/plugins/proxy.rb +16 -2
  28. data/lib/httpx/plugins/proxy/socks4.rb +14 -16
  29. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  30. data/lib/httpx/plugins/push_promise.rb +2 -2
  31. data/lib/httpx/pool.rb +8 -14
  32. data/lib/httpx/request.rb +22 -12
  33. data/lib/httpx/resolver.rb +7 -6
  34. data/lib/httpx/resolver/https.rb +18 -23
  35. data/lib/httpx/resolver/native.rb +22 -19
  36. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  37. data/lib/httpx/resolver/system.rb +3 -3
  38. data/lib/httpx/selector.rb +9 -13
  39. data/lib/httpx/session.rb +24 -21
  40. data/lib/httpx/transcoder.rb +20 -0
  41. data/lib/httpx/transcoder/form.rb +9 -1
  42. data/lib/httpx/version.rb +1 -1
  43. data/sig/connection.rbs +84 -1
  44. data/sig/connection/http1.rbs +66 -0
  45. data/sig/connection/http2.rbs +73 -0
  46. data/sig/headers.rbs +3 -0
  47. data/sig/httpx.rbs +1 -0
  48. data/sig/options.rbs +3 -3
  49. data/sig/plugins/basic_authentication.rbs +1 -1
  50. data/sig/plugins/compression.rbs +1 -1
  51. data/sig/plugins/compression/brotli.rbs +1 -1
  52. data/sig/plugins/compression/deflate.rbs +1 -1
  53. data/sig/plugins/compression/gzip.rbs +1 -1
  54. data/sig/plugins/h2c.rbs +1 -1
  55. data/sig/plugins/multipart.rbs +29 -4
  56. data/sig/plugins/persistent.rbs +1 -1
  57. data/sig/plugins/proxy.rbs +2 -2
  58. data/sig/plugins/proxy/ssh.rbs +1 -1
  59. data/sig/plugins/rate_limiter.rbs +1 -1
  60. data/sig/pool.rbs +36 -2
  61. data/sig/request.rbs +2 -2
  62. data/sig/resolver.rbs +26 -0
  63. data/sig/resolver/https.rbs +51 -0
  64. data/sig/resolver/native.rbs +60 -0
  65. data/sig/resolver/resolver_mixin.rbs +27 -0
  66. data/sig/resolver/system.rbs +17 -0
  67. data/sig/response.rbs +2 -2
  68. data/sig/selector.rbs +20 -0
  69. data/sig/session.rbs +3 -3
  70. data/sig/transcoder.rbs +4 -2
  71. data/sig/transcoder/body.rbs +2 -0
  72. data/sig/transcoder/form.rbs +8 -2
  73. data/sig/transcoder/json.rbs +3 -1
  74. metadata +47 -48
  75. data/lib/httpx/resolver/options.rb +0 -25
  76. data/sig/missing.rbs +0 -12
  77. data/sig/test.rbs +0 -9
@@ -42,12 +42,15 @@ module HTTPX
42
42
  end
43
43
 
44
44
  # deprecated
45
+ # :nocov:
45
46
  def plugins(*args, **opts)
47
+ warn ":#{__method__} is deprecated, use :plugin instead"
46
48
  klass = is_a?(Session) ? self.class : Session
47
49
  klass = Class.new(klass)
48
50
  klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
49
51
  klass.plugins(*args, **opts).new
50
52
  end
53
+ # :nocov:
51
54
 
52
55
  def with(options, &blk)
53
56
  branch(default_options.merge(options), &blk)
@@ -56,7 +59,7 @@ module HTTPX
56
59
  private
57
60
 
58
61
  def default_options
59
- @options || Options.new
62
+ @options || Session.default_options
60
63
  end
61
64
 
62
65
  def branch(options, &blk)
@@ -66,12 +69,10 @@ module HTTPX
66
69
  end
67
70
 
68
71
  def method_missing(meth, *args, **options)
69
- if meth =~ /\Awith_(.+)/
70
- option = Regexp.last_match(1).to_sym
71
- with(option => (args.first || options))
72
- else
73
- super
74
- end
72
+ return super unless meth =~ /\Awith_(.+)/
73
+
74
+ option = Regexp.last_match(1).to_sym
75
+ with(option => (args.first || options))
75
76
  end
76
77
 
77
78
  def respond_to_missing?(meth, *args)
@@ -120,8 +120,8 @@ module HTTPX
120
120
  end
121
121
  end
122
122
 
123
- def create_idle
124
- self.class.new(@type, @origin, @options)
123
+ def create_idle(options = {})
124
+ self.class.new(@type, @origin, @options.merge(options))
125
125
  end
126
126
 
127
127
  def merge(connection)
@@ -131,17 +131,6 @@ module HTTPX
131
131
  end
132
132
  end
133
133
 
134
- def unmerge(connection)
135
- @origins -= connection.instance_variable_get(:@origins)
136
- purge_pending do |request|
137
- request.uri.origin == connection.origin && begin
138
- request.transition(:idle)
139
- connection.send(request)
140
- true
141
- end
142
- end
143
- end
144
-
145
134
  def purge_pending(&block)
146
135
  pendings = []
147
136
  if @parser
@@ -399,7 +388,7 @@ module HTTPX
399
388
  parser.on(:error) do |request, ex|
400
389
  case ex
401
390
  when MisdirectedRequestError
402
- emit(:uncoalesce, request.uri)
391
+ emit(:misdirected, request)
403
392
  else
404
393
  response = ErrorResponse.new(request, ex, @options)
405
394
  request.emit(:response, response)
@@ -435,7 +424,7 @@ module HTTPX
435
424
  remove_instance_variable(:@total_timeout)
436
425
  end
437
426
 
438
- @io.close
427
+ @io.close if @io
439
428
  @read_buffer.clear
440
429
  if @keep_alive_timer
441
430
  @keep_alive_timer.cancel
@@ -21,6 +21,7 @@ module HTTPX
21
21
  @version = [1, 1]
22
22
  @pending = []
23
23
  @requests = []
24
+ @handshake_completed = false
24
25
  end
25
26
 
26
27
  def interests
@@ -155,6 +156,7 @@ module HTTPX
155
156
  @parser.reset!
156
157
  @max_requests -= 1
157
158
  manage_connection(response)
159
+
158
160
  send(@pending.shift) unless @pending.empty?
159
161
  end
160
162
 
@@ -182,17 +184,28 @@ module HTTPX
182
184
  connection = response.headers["connection"]
183
185
  case connection
184
186
  when /keep-alive/i
187
+ if @handshake_completed
188
+ if @max_requests.zero?
189
+ @pending.concat(@requests)
190
+ @requests.clear
191
+ emit(:exhausted)
192
+ end
193
+ return
194
+ end
195
+
185
196
  keep_alive = response.headers["keep-alive"]
186
197
  return unless keep_alive
187
198
 
188
199
  parameters = Hash[keep_alive.split(/ *, */).map do |pair|
189
200
  pair.split(/ *= */)
190
201
  end]
191
- @max_requests = parameters["max"].to_i if parameters.key?("max")
202
+ @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
203
+
192
204
  if parameters.key?("timeout")
193
205
  keep_alive_timeout = parameters["timeout"].to_i
194
206
  emit(:timeout, keep_alive_timeout)
195
207
  end
208
+ @handshake_completed = true
196
209
  when /close/i
197
210
  disable
198
211
  when nil
@@ -51,12 +51,8 @@ module HTTPX
51
51
  :rw
52
52
  end
53
53
 
54
- def reset
55
- init_connection
56
- end
57
-
58
- def close(*args)
59
- @connection.goaway(*args) unless @connection.state == :closed
54
+ def close
55
+ @connection.goaway unless @connection.state == :closed
60
56
  emit(:close)
61
57
  end
62
58
 
@@ -163,14 +159,17 @@ module HTTPX
163
159
  @connection.send_connection_preface
164
160
  end
165
161
 
162
+ alias_method :reset, :init_connection
163
+ public :reset
164
+
166
165
  def handle_stream(stream, request)
167
- stream.on(:close, &method(:on_stream_close).curry[stream, request])
166
+ stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
168
167
  stream.on(:half_close) do
169
168
  log(level: 2) { "#{stream.id}: waiting for response..." }
170
169
  end
171
- stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
172
- stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
173
- stream.on(:data, &method(:on_stream_data).curry[stream, request])
170
+ stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
171
+ stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
172
+ stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
174
173
  end
175
174
 
176
175
  def join_headers(stream, request)
@@ -270,16 +269,16 @@ module HTTPX
270
269
  end
271
270
 
272
271
  def on_close(_last_frame, error, _payload)
272
+ is_connection_closed = @connection.state == :closed
273
273
  if error && error != :no_error
274
+ @buffer.clear if is_connection_closed
274
275
  ex = Error.new(0, error)
275
276
  ex.set_backtrace(caller)
276
- @streams.each_key do |request|
277
- emit(:error, request, ex)
278
- end
277
+ handle_error(ex)
279
278
  end
280
- return unless @connection.state == :closed && @streams.size.zero?
279
+ return unless is_connection_closed && @streams.size.zero?
281
280
 
282
- emit(:close)
281
+ emit(:close, is_connection_closed)
283
282
  end
284
283
 
285
284
  def on_frame_sent(frame)
@@ -317,7 +316,7 @@ module HTTPX
317
316
  end
318
317
 
319
318
  def on_pong(ping)
320
- if !@pings.delete(ping)
319
+ if !@pings.delete(ping.to_s)
321
320
  close(:protocol_error, "ping payload did not match")
322
321
  else
323
322
  emit(:pong)
@@ -139,10 +139,8 @@ module HTTPX
139
139
  elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == DOT
140
140
  # The other is higher
141
141
  -1
142
- elsif othername.end_with?(@hostname) && othername[-@hostname.size - 1, 1] == DOT
143
- # The other is lower
144
- 1
145
142
  else
143
+ # The other is lower
146
144
  1
147
145
  end
148
146
  end
data/lib/httpx/errors.rb CHANGED
@@ -24,6 +24,8 @@ module HTTPX
24
24
 
25
25
  ConnectTimeoutError = Class.new(TimeoutError)
26
26
 
27
+ ResolveTimeoutError = Class.new(TimeoutError)
28
+
27
29
  ResolveError = Class.new(Error)
28
30
 
29
31
  NativeResolveError = Class.new(ResolveError) do
@@ -41,7 +43,7 @@ module HTTPX
41
43
 
42
44
  def initialize(response)
43
45
  @response = response
44
- super("HTTP Error: #{@response.status}")
46
+ super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
45
47
  end
46
48
 
47
49
  def status
data/lib/httpx/headers.rb CHANGED
@@ -119,6 +119,7 @@ module HTTPX
119
119
  def to_hash
120
120
  Hash[to_a]
121
121
  end
122
+ alias_method :to_h, :to_hash
122
123
 
123
124
  # the headers store in array of pairs format
124
125
  def to_a
data/lib/httpx/io/ssl.rb CHANGED
@@ -7,16 +7,16 @@ module HTTPX
7
7
  TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
8
8
  { alpn_protocols: %w[h2 http/1.1] }
9
9
  else
10
- # :nocov:
11
10
  {}
12
- # :nocov:
13
11
  end
14
12
 
15
13
  def initialize(_, _, options)
16
14
  @ctx = OpenSSL::SSL::SSLContext.new
17
15
  ctx_options = TLS_OPTIONS.merge(options.ssl)
16
+ @tls_hostname = ctx_options.delete(:hostname)
18
17
  @ctx.set_params(ctx_options) unless ctx_options.empty?
19
18
  super
19
+ @tls_hostname ||= @hostname
20
20
  @state = :negotiated if @keep_open
21
21
  end
22
22
 
@@ -59,11 +59,11 @@ module HTTPX
59
59
 
60
60
  unless @io.is_a?(OpenSSL::SSL::SSLSocket)
61
61
  @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
62
- @io.hostname = @hostname
62
+ @io.hostname = @tls_hostname
63
63
  @io.sync_close = true
64
64
  end
65
65
  @io.connect_nonblock
66
- @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
66
+ @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
67
67
  transition(:negotiated)
68
68
  rescue ::IO::WaitReadable
69
69
  @interests = :r
@@ -71,7 +71,6 @@ module HTTPX
71
71
  @interests = :w
72
72
  end
73
73
 
74
- # :nocov:
75
74
  if RUBY_VERSION < "2.3"
76
75
  def read(_, buffer)
77
76
  super
@@ -99,14 +98,11 @@ module HTTPX
99
98
  end
100
99
  end
101
100
  end
102
- # :nocov:
103
101
 
104
- # :nocov:
105
102
  def inspect
106
103
  id = @io.closed? ? "closed" : @io.to_io.fileno
107
104
  "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
108
105
  end
109
- # :nocov:
110
106
 
111
107
  private
112
108
 
data/lib/httpx/io/udp.rb CHANGED
@@ -25,7 +25,7 @@ module HTTPX
25
25
  true
26
26
  end
27
27
 
28
- if RUBY_VERSION < "2.2"
28
+ if RUBY_VERSION < "2.3"
29
29
  # :nocov:
30
30
  def close
31
31
  @io.close
@@ -40,14 +40,15 @@ module HTTPX
40
40
  end
41
41
 
42
42
  def write(buffer)
43
- siz = @io.send(buffer, 0, @host, @port)
43
+ siz = @io.send(buffer.to_s, 0, @host, @port)
44
44
  log { "WRITE: #{siz} bytes..." }
45
45
  buffer.shift!(siz)
46
46
  siz
47
47
  end
48
48
 
49
49
  # :nocov:
50
- if RUBY_VERSION < "2.3"
50
+ if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
51
+ RUBY_VERSION < "2.3"
51
52
  def read(size, buffer)
52
53
  data, _ = @io.recvfrom_nonblock(size)
53
54
  buffer.replace(data)
@@ -94,7 +94,7 @@ module HTTPX
94
94
  end
95
95
 
96
96
  def write(chunk)
97
- return super unless defined?(@_inflaters)
97
+ return super unless defined?(@_inflaters) && !chunk.empty?
98
98
 
99
99
  chunk = decompress(chunk)
100
100
  super(chunk)
@@ -107,7 +107,7 @@ module HTTPX
107
107
  case aname
108
108
  when "expires"
109
109
  # RFC 6265 5.2.1
110
- (avalue &&= Time.httpdate(avalue)) || next
110
+ (avalue &&= Time.parse(avalue)) || next
111
111
  when "max-age"
112
112
  # RFC 6265 5.2.2
113
113
  next unless /\A-?\d+\z/.match?(avalue)
@@ -10,6 +10,10 @@ module HTTPX
10
10
  module Expect
11
11
  EXPECT_TIMEOUT = 2
12
12
 
13
+ def self.no_expect_store
14
+ @no_expect_store ||= []
15
+ end
16
+
13
17
  def self.extra_options(options)
14
18
  Class.new(options.class) do
15
19
  def_option(:expect_timeout) do |seconds|
@@ -28,25 +32,46 @@ module HTTPX
28
32
  end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
29
33
  end
30
34
 
31
- module RequestBodyMethods
32
- def initialize(*, options)
35
+ module RequestMethods
36
+ def initialize(*)
33
37
  super
34
- return if @body.nil?
38
+ return if @body.empty?
35
39
 
36
- threshold = options.expect_threshold_size
37
- return if threshold && !unbounded_body? && @body.bytesize < threshold
40
+ threshold = @options.expect_threshold_size
41
+ return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
42
+
43
+ return if Expect.no_expect_store.include?(origin)
38
44
 
39
45
  @headers["expect"] = "100-continue"
40
46
  end
47
+
48
+ def response=(response)
49
+ if response && response.status == 100 &&
50
+ !@headers.key?("expect") &&
51
+ (@state == :body || @state == :done)
52
+
53
+ # if we're past this point, this means that we just received a 100-Continue response,
54
+ # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
55
+ #
56
+ # this means that expect was deactivated for this request too soon, i.e. response took longer.
57
+ #
58
+ # so we have to reactivate it again.
59
+ @headers["expect"] = "100-continue"
60
+ @informational_status = 100
61
+ Expect.no_expect_store.delete(origin)
62
+ end
63
+ super
64
+ end
41
65
  end
42
66
 
43
67
  module ConnectionMethods
44
68
  def send(request)
45
- request.once(:expects) do
69
+ request.once(:expect) do
46
70
  @timers.after(@options.expect_timeout) do
47
- if request.state == :expects && !request.expects?
71
+ if request.state == :expect && !request.expects?
72
+ Expect.no_expect_store << request.origin
48
73
  request.headers.delete("expect")
49
- handle(request)
74
+ consume
50
75
  end
51
76
  end
52
77
  end
@@ -10,38 +10,57 @@ module HTTPX
10
10
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Multipart-Uploads
11
11
  #
12
12
  module Multipart
13
- module FormTranscoder
14
- module_function
15
-
16
- class Encoder
17
- extend Forwardable
18
-
19
- def_delegator :@raw, :content_type
20
-
21
- def_delegator :@raw, :to_s
22
-
23
- def_delegator :@raw, :read
13
+ MULTIPART_VALUE_COND = lambda do |value|
14
+ value.respond_to?(:read) ||
15
+ (value.respond_to?(:to_hash) &&
16
+ value.key?(:body) &&
17
+ (value.key?(:filename) || value.key?(:content_type)))
18
+ end
24
19
 
25
- def initialize(form)
26
- @raw = HTTP::FormData.create(form)
27
- end
20
+ class << self
21
+ def normalize_keys(key, value, &block)
22
+ Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
23
+ end
28
24
 
29
- def bytesize
30
- @raw.content_length
25
+ def load_dependencies(*)
26
+ begin
27
+ unless defined?(HTTP::FormData)
28
+ # in order not to break legacy code, we'll keep loading http/form_data for them.
29
+ require "http/form_data"
30
+ warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
31
+ "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
32
+ "If you'd like to stop seeing this message, require 'http/form_data' yourself."
33
+ end
34
+ rescue LoadError
31
35
  end
36
+ require "httpx/plugins/multipart/encoder"
37
+ require "httpx/plugins/multipart/part"
38
+ require "httpx/plugins/multipart/mime_type_detector"
32
39
  end
33
40
 
34
- def encode(form)
35
- Encoder.new(form)
41
+ def configure(*)
42
+ Transcoder.register("form", FormTranscoder)
36
43
  end
37
44
  end
38
45
 
39
- def self.load_dependencies(*)
40
- require "http/form_data"
41
- end
46
+ module FormTranscoder
47
+ module_function
42
48
 
43
- def self.configure(*)
44
- Transcoder.register("form", FormTranscoder)
49
+ def encode(form)
50
+ if multipart?(form)
51
+ Encoder.new(form)
52
+ else
53
+ Transcoder::Form::Encoder.new(form)
54
+ end
55
+ end
56
+
57
+ def multipart?(data)
58
+ data.any? do |_, v|
59
+ MULTIPART_VALUE_COND.call(v) ||
60
+ (v.respond_to?(:to_ary) && v.to_ary.any?(&MULTIPART_VALUE_COND)) ||
61
+ (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| MULTIPART_VALUE_COND.call(e) })
62
+ end
63
+ end
45
64
  end
46
65
  end
47
66
  register_plugin :multipart, Multipart