httpx 0.10.1 → 0.11.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -3
  3. data/doc/release_notes/0_10_1.md +1 -3
  4. data/doc/release_notes/0_10_2.md +5 -0
  5. data/doc/release_notes/0_11_0.md +76 -0
  6. data/doc/release_notes/0_11_1.md +1 -0
  7. data/doc/release_notes/0_11_2.md +5 -0
  8. data/doc/release_notes/0_11_3.md +5 -0
  9. data/lib/httpx/adapters/datadog.rb +205 -0
  10. data/lib/httpx/adapters/faraday.rb +0 -2
  11. data/lib/httpx/adapters/webmock.rb +123 -0
  12. data/lib/httpx/chainable.rb +1 -1
  13. data/lib/httpx/connection/http1.rb +10 -0
  14. data/lib/httpx/connection/http2.rb +4 -4
  15. data/lib/httpx/domain_name.rb +1 -3
  16. data/lib/httpx/errors.rb +2 -0
  17. data/lib/httpx/headers.rb +1 -0
  18. data/lib/httpx/io/ssl.rb +4 -8
  19. data/lib/httpx/io/udp.rb +4 -3
  20. data/lib/httpx/plugins/compression.rb +1 -1
  21. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  22. data/lib/httpx/plugins/expect.rb +33 -8
  23. data/lib/httpx/plugins/multipart.rb +40 -35
  24. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  25. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  26. data/lib/httpx/plugins/multipart/part.rb +34 -0
  27. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  28. data/lib/httpx/plugins/push_promise.rb +2 -2
  29. data/lib/httpx/request.rb +21 -11
  30. data/lib/httpx/resolver.rb +7 -4
  31. data/lib/httpx/resolver/https.rb +4 -2
  32. data/lib/httpx/resolver/native.rb +10 -6
  33. data/lib/httpx/resolver/system.rb +1 -1
  34. data/lib/httpx/selector.rb +1 -0
  35. data/lib/httpx/session.rb +15 -18
  36. data/lib/httpx/transcoder.rb +6 -4
  37. data/lib/httpx/version.rb +1 -1
  38. data/sig/connection/http2.rbs +3 -4
  39. data/sig/headers.rbs +3 -0
  40. data/sig/plugins/multipart.rbs +27 -4
  41. data/sig/request.rbs +1 -1
  42. data/sig/resolver/https.rbs +2 -0
  43. data/sig/response.rbs +1 -1
  44. data/sig/session.rbs +1 -1
  45. data/sig/transcoder.rbs +2 -2
  46. data/sig/transcoder/body.rbs +2 -0
  47. data/sig/transcoder/form.rbs +7 -1
  48. data/sig/transcoder/json.rbs +3 -1
  49. metadata +40 -46
  50. data/sig/missing.rbs +0 -12
@@ -59,7 +59,7 @@ module HTTPX
59
59
  private
60
60
 
61
61
  def default_options
62
- @options || Options.new
62
+ @options || Session.default_options
63
63
  end
64
64
 
65
65
  def branch(options, &blk)
@@ -161,6 +161,16 @@ module HTTPX
161
161
  end
162
162
 
163
163
  def handle_error(ex)
164
+ if ex.is_a?(EOFError) && @request && @request.response &&
165
+ !@request.response.headers.key?("content-length") &&
166
+ !@request.response.headers.key?("transfer-encoding")
167
+ # if the response does not contain a content-length header, the server closing the
168
+ # connnection is the indicator of response consumed.
169
+ # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
170
+ on_complete
171
+ return
172
+ end
173
+
164
174
  if @pipelining
165
175
  disable
166
176
  else
@@ -163,13 +163,13 @@ module HTTPX
163
163
  public :reset
164
164
 
165
165
  def handle_stream(stream, request)
166
- stream.on(:close, &method(:on_stream_close).curry[stream, request])
166
+ stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
167
167
  stream.on(:half_close) do
168
168
  log(level: 2) { "#{stream.id}: waiting for response..." }
169
169
  end
170
- stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
171
- stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
172
- stream.on(:data, &method(:on_stream_data).curry[stream, request])
170
+ stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
171
+ stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
172
+ stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
173
173
  end
174
174
 
175
175
  def join_headers(stream, request)
@@ -139,10 +139,8 @@ module HTTPX
139
139
  elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == DOT
140
140
  # The other is higher
141
141
  -1
142
- elsif othername.end_with?(@hostname) && othername[-@hostname.size - 1, 1] == DOT
143
- # The other is lower
144
- 1
145
142
  else
143
+ # The other is lower
146
144
  1
147
145
  end
148
146
  end
data/lib/httpx/errors.rb CHANGED
@@ -24,6 +24,8 @@ module HTTPX
24
24
 
25
25
  ConnectTimeoutError = Class.new(TimeoutError)
26
26
 
27
+ ResolveTimeoutError = Class.new(TimeoutError)
28
+
27
29
  ResolveError = Class.new(Error)
28
30
 
29
31
  NativeResolveError = Class.new(ResolveError) do
data/lib/httpx/headers.rb CHANGED
@@ -119,6 +119,7 @@ module HTTPX
119
119
  def to_hash
120
120
  Hash[to_a]
121
121
  end
122
+ alias_method :to_h, :to_hash
122
123
 
123
124
  # the headers store in array of pairs format
124
125
  def to_a
data/lib/httpx/io/ssl.rb CHANGED
@@ -7,16 +7,16 @@ module HTTPX
7
7
  TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
8
8
  { alpn_protocols: %w[h2 http/1.1] }
9
9
  else
10
- # :nocov:
11
10
  {}
12
- # :nocov:
13
11
  end
14
12
 
15
13
  def initialize(_, _, options)
16
14
  @ctx = OpenSSL::SSL::SSLContext.new
17
15
  ctx_options = TLS_OPTIONS.merge(options.ssl)
16
+ @tls_hostname = ctx_options.delete(:hostname)
18
17
  @ctx.set_params(ctx_options) unless ctx_options.empty?
19
18
  super
19
+ @tls_hostname ||= @hostname
20
20
  @state = :negotiated if @keep_open
21
21
  end
22
22
 
@@ -59,11 +59,11 @@ module HTTPX
59
59
 
60
60
  unless @io.is_a?(OpenSSL::SSL::SSLSocket)
61
61
  @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
62
- @io.hostname = @hostname
62
+ @io.hostname = @tls_hostname
63
63
  @io.sync_close = true
64
64
  end
65
65
  @io.connect_nonblock
66
- @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
66
+ @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
67
67
  transition(:negotiated)
68
68
  rescue ::IO::WaitReadable
69
69
  @interests = :r
@@ -71,7 +71,6 @@ module HTTPX
71
71
  @interests = :w
72
72
  end
73
73
 
74
- # :nocov:
75
74
  if RUBY_VERSION < "2.3"
76
75
  def read(_, buffer)
77
76
  super
@@ -99,14 +98,11 @@ module HTTPX
99
98
  end
100
99
  end
101
100
  end
102
- # :nocov:
103
101
 
104
- # :nocov:
105
102
  def inspect
106
103
  id = @io.closed? ? "closed" : @io.to_io.fileno
107
104
  "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
108
105
  end
109
- # :nocov:
110
106
 
111
107
  private
112
108
 
data/lib/httpx/io/udp.rb CHANGED
@@ -25,7 +25,7 @@ module HTTPX
25
25
  true
26
26
  end
27
27
 
28
- if RUBY_VERSION < "2.2"
28
+ if RUBY_VERSION < "2.3"
29
29
  # :nocov:
30
30
  def close
31
31
  @io.close
@@ -40,14 +40,15 @@ module HTTPX
40
40
  end
41
41
 
42
42
  def write(buffer)
43
- siz = @io.send(buffer, 0, @host, @port)
43
+ siz = @io.send(buffer.to_s, 0, @host, @port)
44
44
  log { "WRITE: #{siz} bytes..." }
45
45
  buffer.shift!(siz)
46
46
  siz
47
47
  end
48
48
 
49
49
  # :nocov:
50
- if RUBY_VERSION < "2.3"
50
+ if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
51
+ RUBY_VERSION < "2.3"
51
52
  def read(size, buffer)
52
53
  data, _ = @io.recvfrom_nonblock(size)
53
54
  buffer.replace(data)
@@ -94,7 +94,7 @@ module HTTPX
94
94
  end
95
95
 
96
96
  def write(chunk)
97
- return super unless defined?(@_inflaters)
97
+ return super unless defined?(@_inflaters) && !chunk.empty?
98
98
 
99
99
  chunk = decompress(chunk)
100
100
  super(chunk)
@@ -107,7 +107,7 @@ module HTTPX
107
107
  case aname
108
108
  when "expires"
109
109
  # RFC 6265 5.2.1
110
- (avalue &&= Time.httpdate(avalue)) || next
110
+ (avalue &&= Time.parse(avalue)) || next
111
111
  when "max-age"
112
112
  # RFC 6265 5.2.2
113
113
  next unless /\A-?\d+\z/.match?(avalue)
@@ -10,6 +10,10 @@ module HTTPX
10
10
  module Expect
11
11
  EXPECT_TIMEOUT = 2
12
12
 
13
+ def self.no_expect_store
14
+ @no_expect_store ||= []
15
+ end
16
+
13
17
  def self.extra_options(options)
14
18
  Class.new(options.class) do
15
19
  def_option(:expect_timeout) do |seconds|
@@ -28,25 +32,46 @@ module HTTPX
28
32
  end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
29
33
  end
30
34
 
31
- module RequestBodyMethods
32
- def initialize(*, options)
35
+ module RequestMethods
36
+ def initialize(*)
33
37
  super
34
- return if @body.nil?
38
+ return if @body.empty?
35
39
 
36
- threshold = options.expect_threshold_size
37
- return if threshold && !unbounded_body? && @body.bytesize < threshold
40
+ threshold = @options.expect_threshold_size
41
+ return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
42
+
43
+ return if Expect.no_expect_store.include?(origin)
38
44
 
39
45
  @headers["expect"] = "100-continue"
40
46
  end
47
+
48
+ def response=(response)
49
+ if response && response.status == 100 &&
50
+ !@headers.key?("expect") &&
51
+ (@state == :body || @state == :done)
52
+
53
+ # if we're past this point, this means that we just received a 100-Continue response,
54
+ # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
55
+ #
56
+ # this means that expect was deactivated for this request too soon, i.e. response took longer.
57
+ #
58
+ # so we have to reactivate it again.
59
+ @headers["expect"] = "100-continue"
60
+ @informational_status = 100
61
+ Expect.no_expect_store.delete(origin)
62
+ end
63
+ super
64
+ end
41
65
  end
42
66
 
43
67
  module ConnectionMethods
44
68
  def send(request)
45
- request.once(:expects) do
69
+ request.once(:expect) do
46
70
  @timers.after(@options.expect_timeout) do
47
- if request.state == :expects && !request.expects?
71
+ if request.state == :expect && !request.expects?
72
+ Expect.no_expect_store << request.origin
48
73
  request.headers.delete("expect")
49
- handle(request)
74
+ consume
50
75
  end
51
76
  end
52
77
  end
@@ -10,53 +10,58 @@ module HTTPX
10
10
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Multipart-Uploads
11
11
  #
12
12
  module Multipart
13
- module FormTranscoder
14
- module_function
15
-
16
- class Encoder
17
- extend Forwardable
18
-
19
- def_delegator :@raw, :content_type
20
-
21
- def_delegator :@raw, :to_s
13
+ MULTIPART_VALUE_COND = lambda do |value|
14
+ value.respond_to?(:read) ||
15
+ (value.respond_to?(:to_hash) &&
16
+ value.key?(:body) &&
17
+ (value.key?(:filename) || value.key?(:content_type)))
18
+ end
22
19
 
23
- def_delegator :@raw, :read
20
+ class << self
21
+ def normalize_keys(key, value, &block)
22
+ Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
23
+ end
24
24
 
25
- def initialize(form)
26
- @raw = if multipart?(form)
27
- HTTP::FormData::Multipart.new(Hash[*form.map { |k, v| Transcoder.enum_for(:normalize_keys, k, v).to_a }])
28
- else
29
- HTTP::FormData::Urlencoded.new(form, :encoder => Transcoder::Form.method(:encode))
25
+ def load_dependencies(*)
26
+ begin
27
+ unless defined?(HTTP::FormData)
28
+ # in order not to break legacy code, we'll keep loading http/form_data for them.
29
+ require "http/form_data"
30
+ warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
31
+ "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
32
+ "If you'd like to stop seeing this message, require 'http/form_data' yourself."
30
33
  end
34
+ rescue LoadError
31
35
  end
36
+ require "httpx/plugins/multipart/encoder"
37
+ require "httpx/plugins/multipart/part"
38
+ require "httpx/plugins/multipart/mime_type_detector"
39
+ end
32
40
 
33
- def bytesize
34
- @raw.content_length
35
- end
41
+ def configure(*)
42
+ Transcoder.register("form", FormTranscoder)
43
+ end
44
+ end
36
45
 
37
- private
46
+ module FormTranscoder
47
+ module_function
38
48
 
39
- def multipart?(data)
40
- data.any? do |_, v|
41
- v.is_a?(HTTP::FormData::Part) ||
42
- (v.respond_to?(:to_ary) && v.to_ary.any? { |e| e.is_a?(HTTP::FormData::Part) }) ||
43
- (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| e.is_a?(HTTP::FormData::Part) })
44
- end
49
+ def encode(form)
50
+ if multipart?(form)
51
+ Encoder.new(form)
52
+ else
53
+ Transcoder::Form::Encoder.new(form)
45
54
  end
46
55
  end
47
56
 
48
- def encode(form)
49
- Encoder.new(form)
57
+ def multipart?(data)
58
+ data.any? do |_, v|
59
+ MULTIPART_VALUE_COND.call(v) ||
60
+ (v.respond_to?(:to_ary) && v.to_ary.any?(&MULTIPART_VALUE_COND)) ||
61
+ (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| MULTIPART_VALUE_COND.call(e) })
62
+ end
50
63
  end
51
64
  end
52
-
53
- def self.load_dependencies(*)
54
- require "http/form_data"
55
- end
56
-
57
- def self.configure(*)
58
- Transcoder.register("form", FormTranscoder)
59
- end
60
65
  end
61
66
  register_plugin :multipart, Multipart
62
67
  end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX::Plugins
4
+ module Multipart
5
+ class Encoder
6
+ attr_reader :bytesize
7
+
8
+ def initialize(form)
9
+ @boundary = ("-" * 21) << SecureRandom.hex(21)
10
+ @part_index = 0
11
+ @buffer = "".b
12
+
13
+ @form = form
14
+ @parts = to_parts(form)
15
+ end
16
+
17
+ def content_type
18
+ "multipart/form-data; boundary=#{@boundary}"
19
+ end
20
+
21
+ def read(length = nil, outbuf = nil)
22
+ data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
23
+ data ||= "".b
24
+
25
+ read_chunks(data, length)
26
+
27
+ data unless length && data.empty?
28
+ end
29
+
30
+ def rewind
31
+ form = @form.each_with_object([]) do |(key, val), aux|
32
+ v = case val
33
+ when File
34
+ val = val.reopen(val.path, File::RDONLY) if val.closed?
35
+ val.rewind
36
+ val
37
+ else
38
+ v
39
+ end
40
+ aux << [key, v]
41
+ end
42
+ @form = form
43
+ @parts = to_parts(form)
44
+ end
45
+
46
+ private
47
+
48
+ def to_parts(form)
49
+ @bytesize = 0
50
+ params = form.each_with_object([]) do |(key, val), aux|
51
+ Multipart.normalize_keys(key, val) do |k, v|
52
+ next if v.nil?
53
+
54
+ value, content_type, filename = Part.call(v)
55
+
56
+ header = header_part(k, content_type, filename)
57
+ @bytesize += header.size
58
+ aux << header
59
+
60
+ @bytesize += value.size
61
+ aux << value
62
+
63
+ delimiter = StringIO.new("\r\n")
64
+ @bytesize += delimiter.size
65
+ aux << delimiter
66
+ end
67
+ end
68
+ final_delimiter = StringIO.new("--#{@boundary}--\r\n")
69
+ @bytesize += final_delimiter.size
70
+ params << final_delimiter
71
+
72
+ params
73
+ end
74
+
75
+ def header_part(key, content_type, filename)
76
+ header = "--#{@boundary}\r\n".b
77
+ header << "Content-Disposition: form-data; name=#{key.inspect}".b
78
+ header << "; filename=#{filename.inspect}" if filename
79
+ header << "\r\nContent-Type: #{content_type}\r\n\r\n"
80
+ StringIO.new(header)
81
+ end
82
+
83
+ def read_chunks(buffer, length = nil)
84
+ while @part_index < @parts.size
85
+ chunk = read_from_part(length)
86
+
87
+ next unless chunk
88
+
89
+ buffer << chunk.force_encoding(Encoding::BINARY)
90
+
91
+ next unless length
92
+
93
+ length -= chunk.bytesize
94
+
95
+ break if length.zero?
96
+ end
97
+ end
98
+
99
+ # if there's a current part to read from, tries to read a chunk.
100
+ def read_from_part(max_length = nil)
101
+ part = @parts[@part_index]
102
+
103
+ chunk = part.read(max_length, @buffer)
104
+
105
+ return chunk if chunk && !chunk.empty?
106
+
107
+ part.close if part.respond_to?(:close)
108
+
109
+ @part_index += 1
110
+
111
+ nil
112
+ end
113
+ end
114
+ end
115
+ end