httpx 0.9.0 → 0.11.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +13 -3
  4. data/doc/release_notes/0_10_0.md +66 -0
  5. data/doc/release_notes/0_10_1.md +37 -0
  6. data/doc/release_notes/0_10_2.md +5 -0
  7. data/doc/release_notes/0_11_0.md +76 -0
  8. data/doc/release_notes/0_11_1.md +1 -0
  9. data/lib/httpx.rb +2 -0
  10. data/lib/httpx/adapters/datadog.rb +205 -0
  11. data/lib/httpx/adapters/faraday.rb +1 -3
  12. data/lib/httpx/adapters/webmock.rb +123 -0
  13. data/lib/httpx/chainable.rb +10 -9
  14. data/lib/httpx/connection.rb +7 -24
  15. data/lib/httpx/connection/http1.rb +15 -2
  16. data/lib/httpx/connection/http2.rb +15 -16
  17. data/lib/httpx/domain_name.rb +438 -0
  18. data/lib/httpx/errors.rb +4 -1
  19. data/lib/httpx/extensions.rb +21 -1
  20. data/lib/httpx/headers.rb +1 -0
  21. data/lib/httpx/io/ssl.rb +4 -9
  22. data/lib/httpx/io/tcp.rb +6 -5
  23. data/lib/httpx/io/udp.rb +8 -4
  24. data/lib/httpx/options.rb +2 -0
  25. data/lib/httpx/parser/http1.rb +14 -17
  26. data/lib/httpx/plugins/compression.rb +28 -63
  27. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  28. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  29. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  30. data/lib/httpx/plugins/cookies.rb +21 -60
  31. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  32. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  33. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  34. data/lib/httpx/plugins/expect.rb +34 -11
  35. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  36. data/lib/httpx/plugins/h2c.rb +1 -1
  37. data/lib/httpx/plugins/multipart.rb +41 -30
  38. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  39. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  40. data/lib/httpx/plugins/multipart/part.rb +34 -0
  41. data/lib/httpx/plugins/persistent.rb +6 -1
  42. data/lib/httpx/plugins/proxy.rb +16 -2
  43. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  44. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  45. data/lib/httpx/plugins/push_promise.rb +2 -2
  46. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  47. data/lib/httpx/plugins/retries.rb +3 -2
  48. data/lib/httpx/plugins/stream.rb +109 -13
  49. data/lib/httpx/pool.rb +14 -20
  50. data/lib/httpx/request.rb +29 -31
  51. data/lib/httpx/resolver.rb +7 -6
  52. data/lib/httpx/resolver/https.rb +25 -25
  53. data/lib/httpx/resolver/native.rb +29 -22
  54. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  55. data/lib/httpx/resolver/system.rb +3 -3
  56. data/lib/httpx/response.rb +16 -23
  57. data/lib/httpx/selector.rb +11 -17
  58. data/lib/httpx/session.rb +39 -30
  59. data/lib/httpx/transcoder.rb +20 -0
  60. data/lib/httpx/transcoder/chunker.rb +0 -2
  61. data/lib/httpx/transcoder/form.rb +9 -7
  62. data/lib/httpx/transcoder/json.rb +0 -4
  63. data/lib/httpx/utils.rb +45 -0
  64. data/lib/httpx/version.rb +1 -1
  65. data/sig/buffer.rbs +24 -0
  66. data/sig/callbacks.rbs +14 -0
  67. data/sig/chainable.rbs +37 -0
  68. data/sig/connection.rbs +85 -0
  69. data/sig/connection/http1.rbs +66 -0
  70. data/sig/connection/http2.rbs +77 -0
  71. data/sig/domain_name.rbs +17 -0
  72. data/sig/errors.rbs +3 -0
  73. data/sig/headers.rbs +45 -0
  74. data/sig/httpx.rbs +15 -0
  75. data/sig/loggable.rbs +11 -0
  76. data/sig/options.rbs +118 -0
  77. data/sig/parser/http1.rbs +50 -0
  78. data/sig/plugins/authentication.rbs +11 -0
  79. data/sig/plugins/basic_authentication.rbs +13 -0
  80. data/sig/plugins/compression.rbs +55 -0
  81. data/sig/plugins/compression/brotli.rbs +21 -0
  82. data/sig/plugins/compression/deflate.rbs +17 -0
  83. data/sig/plugins/compression/gzip.rbs +29 -0
  84. data/sig/plugins/cookies.rbs +26 -0
  85. data/sig/plugins/cookies/cookie.rbs +50 -0
  86. data/sig/plugins/cookies/jar.rbs +27 -0
  87. data/sig/plugins/digest_authentication.rbs +33 -0
  88. data/sig/plugins/expect.rbs +19 -0
  89. data/sig/plugins/follow_redirects.rbs +37 -0
  90. data/sig/plugins/h2c.rbs +26 -0
  91. data/sig/plugins/multipart.rbs +44 -0
  92. data/sig/plugins/persistent.rbs +17 -0
  93. data/sig/plugins/proxy.rbs +47 -0
  94. data/sig/plugins/proxy/http.rbs +14 -0
  95. data/sig/plugins/proxy/socks4.rbs +33 -0
  96. data/sig/plugins/proxy/socks5.rbs +36 -0
  97. data/sig/plugins/proxy/ssh.rbs +18 -0
  98. data/sig/plugins/push_promise.rbs +22 -0
  99. data/sig/plugins/rate_limiter.rbs +11 -0
  100. data/sig/plugins/retries.rbs +48 -0
  101. data/sig/plugins/stream.rbs +39 -0
  102. data/sig/pool.rbs +36 -0
  103. data/sig/registry.rbs +9 -0
  104. data/sig/request.rbs +61 -0
  105. data/sig/resolver.rbs +26 -0
  106. data/sig/resolver/https.rbs +51 -0
  107. data/sig/resolver/native.rbs +60 -0
  108. data/sig/resolver/resolver_mixin.rbs +27 -0
  109. data/sig/resolver/system.rbs +17 -0
  110. data/sig/response.rbs +87 -0
  111. data/sig/selector.rbs +20 -0
  112. data/sig/session.rbs +49 -0
  113. data/sig/timeout.rbs +29 -0
  114. data/sig/transcoder.rbs +18 -0
  115. data/sig/transcoder/body.rbs +20 -0
  116. data/sig/transcoder/chunker.rbs +32 -0
  117. data/sig/transcoder/form.rbs +22 -0
  118. data/sig/transcoder/json.rbs +16 -0
  119. metadata +99 -59
  120. data/lib/httpx/resolver/options.rb +0 -25
@@ -24,10 +24,13 @@ module HTTPX
24
24
 
25
25
  ConnectTimeoutError = Class.new(TimeoutError)
26
26
 
27
+ ResolveTimeoutError = Class.new(TimeoutError)
28
+
27
29
  ResolveError = Class.new(Error)
28
30
 
29
31
  NativeResolveError = Class.new(ResolveError) do
30
32
  attr_reader :connection, :host
33
+
31
34
  def initialize(connection, host, message = "Can't resolve #{host}")
32
35
  @connection = connection
33
36
  @host = host
@@ -40,7 +43,7 @@ module HTTPX
40
43
 
41
44
  def initialize(response)
42
45
  @response = response
43
- super("HTTP Error: #{@response.status}")
46
+ super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
44
47
  end
45
48
 
46
49
  def status
@@ -54,11 +54,31 @@ module HTTPX
54
54
  Numeric.__send__(:include, NegMethods)
55
55
  end
56
56
 
57
+ module RegexpExtensions
58
+ # If you wonder why this is there: the oauth feature uses a refinement to enhance the
59
+ # Regexp class locally with #match? , but this is never tested, because ActiveSupport
60
+ # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
61
+ # :nocov:
62
+ refine(Regexp) do
63
+ def match?(*args)
64
+ !match(*args).nil?
65
+ end
66
+ end
67
+ end
68
+
57
69
  module URIExtensions
58
70
  refine URI::Generic do
71
+ def non_ascii_hostname
72
+ @non_ascii_hostname
73
+ end
74
+
75
+ def non_ascii_hostname=(hostname)
76
+ @non_ascii_hostname = hostname
77
+ end
78
+
59
79
  def authority
60
80
  port_string = port == default_port ? nil : ":#{port}"
61
- "#{host}#{port_string}"
81
+ "#{@non_ascii_hostname || host}#{port_string}"
62
82
  end
63
83
 
64
84
  def origin
@@ -119,6 +119,7 @@ module HTTPX
119
119
  def to_hash
120
120
  Hash[to_a]
121
121
  end
122
+ alias_method :to_h, :to_hash
122
123
 
123
124
  # the headers store in array of pairs format
124
125
  def to_a
@@ -7,16 +7,16 @@ module HTTPX
7
7
  TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
8
8
  { alpn_protocols: %w[h2 http/1.1] }
9
9
  else
10
- # :nocov:
11
10
  {}
12
- # :nocov:
13
11
  end
14
12
 
15
13
  def initialize(_, _, options)
16
14
  @ctx = OpenSSL::SSL::SSLContext.new
17
15
  ctx_options = TLS_OPTIONS.merge(options.ssl)
16
+ @tls_hostname = ctx_options.delete(:hostname)
18
17
  @ctx.set_params(ctx_options) unless ctx_options.empty?
19
18
  super
19
+ @tls_hostname ||= @hostname
20
20
  @state = :negotiated if @keep_open
21
21
  end
22
22
 
@@ -42,7 +42,6 @@ module HTTPX
42
42
  # allow reconnections
43
43
  # connect only works if initial @io is a socket
44
44
  @io = @io.io if @io.respond_to?(:io)
45
- @negotiated = false
46
45
  end
47
46
 
48
47
  def connected?
@@ -60,11 +59,11 @@ module HTTPX
60
59
 
61
60
  unless @io.is_a?(OpenSSL::SSL::SSLSocket)
62
61
  @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
63
- @io.hostname = @hostname
62
+ @io.hostname = @tls_hostname
64
63
  @io.sync_close = true
65
64
  end
66
65
  @io.connect_nonblock
67
- @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
66
+ @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
68
67
  transition(:negotiated)
69
68
  rescue ::IO::WaitReadable
70
69
  @interests = :r
@@ -72,7 +71,6 @@ module HTTPX
72
71
  @interests = :w
73
72
  end
74
73
 
75
- # :nocov:
76
74
  if RUBY_VERSION < "2.3"
77
75
  def read(_, buffer)
78
76
  super
@@ -100,14 +98,11 @@ module HTTPX
100
98
  end
101
99
  end
102
100
  end
103
- # :nocov:
104
101
 
105
- # :nocov:
106
102
  def inspect
107
103
  id = @io.closed? ? "closed" : @io.to_io.fileno
108
104
  "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
109
105
  end
110
- # :nocov:
111
106
 
112
107
  private
113
108
 
@@ -7,11 +7,7 @@ module HTTPX
7
7
  class TCP
8
8
  include Loggable
9
9
 
10
- attr_reader :ip, :port
11
-
12
- attr_reader :addresses
13
-
14
- attr_reader :state
10
+ attr_reader :ip, :port, :addresses, :state
15
11
 
16
12
  alias_method :host, :ip
17
13
 
@@ -86,6 +82,7 @@ module HTTPX
86
82
  # :nocov:
87
83
  def read(size, buffer)
88
84
  @io.read_nonblock(size, buffer)
85
+ log { "READ: #{buffer.bytesize} bytes..." }
89
86
  buffer.bytesize
90
87
  rescue ::IO::WaitReadable
91
88
  buffer.clear
@@ -96,6 +93,7 @@ module HTTPX
96
93
 
97
94
  def write(buffer)
98
95
  siz = @io.write_nonblock(buffer)
96
+ log { "WRITE: #{siz} bytes..." }
99
97
  buffer.shift!(siz)
100
98
  siz
101
99
  rescue ::IO::WaitWritable
@@ -113,6 +111,7 @@ module HTTPX
113
111
  end
114
112
  return if ret.nil?
115
113
 
114
+ log { "READ: #{buffer.bytesize} bytes..." }
116
115
  buffer.bytesize
117
116
  end
118
117
 
@@ -121,6 +120,8 @@ module HTTPX
121
120
  return 0 if siz == :wait_writable
122
121
  return if siz.nil?
123
122
 
123
+ log { "WRITE: #{siz} bytes..." }
124
+
124
125
  buffer.shift!(siz)
125
126
  siz
126
127
  end
@@ -7,11 +7,12 @@ module HTTPX
7
7
  class UDP
8
8
  include Loggable
9
9
 
10
- def initialize(uri, _, _)
10
+ def initialize(uri, _, options)
11
11
  ip = IPAddr.new(uri.host)
12
12
  @host = ip.to_s
13
13
  @port = uri.port
14
14
  @io = UDPSocket.new(ip.family)
15
+ @options = options
15
16
  end
16
17
 
17
18
  def to_io
@@ -24,7 +25,7 @@ module HTTPX
24
25
  true
25
26
  end
26
27
 
27
- if RUBY_VERSION < "2.2"
28
+ if RUBY_VERSION < "2.3"
28
29
  # :nocov:
29
30
  def close
30
31
  @io.close
@@ -39,16 +40,19 @@ module HTTPX
39
40
  end
40
41
 
41
42
  def write(buffer)
42
- siz = @io.send(buffer, 0, @host, @port)
43
+ siz = @io.send(buffer.to_s, 0, @host, @port)
44
+ log { "WRITE: #{siz} bytes..." }
43
45
  buffer.shift!(siz)
44
46
  siz
45
47
  end
46
48
 
47
49
  # :nocov:
48
- if RUBY_VERSION < "2.3"
50
+ if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
51
+ RUBY_VERSION < "2.3"
49
52
  def read(size, buffer)
50
53
  data, _ = @io.recvfrom_nonblock(size)
51
54
  buffer.replace(data)
55
+ log { "READ: #{buffer.bytesize} bytes..." }
52
56
  buffer.bytesize
53
57
  rescue ::IO::WaitReadable
54
58
  0
@@ -76,6 +76,8 @@ module HTTPX
76
76
 
77
77
  defaults.merge!(options)
78
78
  defaults.each do |(k, v)|
79
+ next if v.nil?
80
+
79
81
  __send__(:"#{k}=", v)
80
82
  end
81
83
  end
@@ -9,10 +9,9 @@ module HTTPX
9
9
 
10
10
  attr_reader :status_code, :http_version, :headers
11
11
 
12
- def initialize(observer, header_separator: ":")
12
+ def initialize(observer)
13
13
  @observer = observer
14
14
  @state = :idle
15
- @header_separator = header_separator
16
15
  @buffer = "".b
17
16
  @headers = {}
18
17
  end
@@ -40,25 +39,25 @@ module HTTPX
40
39
  private
41
40
 
42
41
  def parse
43
- state = @state
44
- case @state
45
- when :idle
46
- parse_headline
47
- when :headers
48
- parse_headers
49
- when :trailers
50
- parse_headers
51
- when :data
52
- parse_data
42
+ loop do
43
+ state = @state
44
+ case @state
45
+ when :idle
46
+ parse_headline
47
+ when :headers, :trailers
48
+ parse_headers
49
+ when :data
50
+ parse_data
51
+ end
52
+ return if @buffer.empty? || state == @state
53
53
  end
54
- parse if !@buffer.empty? && state != @state
55
54
  end
56
55
 
57
56
  def parse_headline
58
57
  idx = @buffer.index("\n")
59
58
  return unless idx
60
59
 
61
- (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
60
+ (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
62
61
  raise(Error, "wrong head line format")
63
62
  version, code, _ = m.captures
64
63
  raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
@@ -91,12 +90,10 @@ module HTTPX
91
90
  @observer.on_trailers(headers)
92
91
  headers.clear
93
92
  nextstate(:complete)
94
- else
95
- raise Error, "wrong header format"
96
93
  end
97
94
  return
98
95
  end
99
- separator_index = line.index(@header_separator)
96
+ separator_index = line.index(":")
100
97
  raise Error, "wrong header format" unless separator_index
101
98
 
102
99
  key = line[0..separator_index - 1]
@@ -46,16 +46,13 @@ module HTTPX
46
46
  super
47
47
  return if @body.nil?
48
48
 
49
- if (threshold = options.compression_threshold_size)
50
- unless unbounded_body?
51
- return if @body.bytesize < threshold
52
- end
53
- end
49
+ threshold = options.compression_threshold_size
50
+ return if threshold && !unbounded_body? && @body.bytesize < threshold
54
51
 
55
52
  @headers.get("content-encoding").each do |encoding|
56
53
  next if encoding == "identity"
57
54
 
58
- @body = Encoder.new(@body, Compression.registry(encoding).encoder)
55
+ @body = Encoder.new(@body, Compression.registry(encoding).deflater)
59
56
  end
60
57
  @headers["content-length"] = @body.bytesize unless chunked?
61
58
  end
@@ -71,57 +68,53 @@ module HTTPX
71
68
 
72
69
  return unless @headers.key?("content-encoding")
73
70
 
74
- @_decoders = @headers.get("content-encoding").map do |encoding|
71
+ # remove encodings that we are able to decode
72
+ @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
73
+
74
+ compressed_length = if @headers.key?("content-length")
75
+ @headers["content-length"].to_i
76
+ else
77
+ Float::INFINITY
78
+ end
79
+
80
+ @_inflaters = @headers.get("content-encoding").map do |encoding|
75
81
  next if encoding == "identity"
76
82
 
77
- decoder = Compression.registry(encoding).decoder
83
+ inflater = Compression.registry(encoding).inflater(compressed_length)
78
84
  # do not uncompress if there is no decoder available. In fact, we can't reliably
79
85
  # continue decompressing beyond that, so ignore.
80
- break unless decoder
86
+ break unless inflater
81
87
 
82
88
  @encodings << encoding
83
- decoder
89
+ inflater
84
90
  end.compact
85
91
 
86
- # remove encodings that we are able to decode
87
- @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
88
-
89
- @_compressed_length = if @headers.key?("content-length")
90
- @headers["content-length"].to_i
91
- else
92
- Float::INFINITY
93
- end
92
+ # this can happen if the only declared encoding is "identity"
93
+ remove_instance_variable(:@_inflaters) if @_inflaters.empty?
94
94
  end
95
95
 
96
96
  def write(chunk)
97
- return super unless defined?(@_compressed_length)
97
+ return super unless defined?(@_inflaters) && !chunk.empty?
98
98
 
99
- @_compressed_length -= chunk.bytesize
100
99
  chunk = decompress(chunk)
101
100
  super(chunk)
102
101
  end
103
102
 
104
- def close
105
- super
106
-
107
- return unless defined?(@_decoders)
108
-
109
- @_decoders.each(&:close)
110
- end
111
-
112
103
  private
113
104
 
114
105
  def decompress(buffer)
115
- @_decoders.reverse_each do |decoder|
116
- buffer = decoder.decode(buffer)
117
- buffer << decoder.finish if @_compressed_length <= 0
106
+ @_inflaters.reverse_each do |inflater|
107
+ buffer = inflater.inflate(buffer)
118
108
  end
119
109
  buffer
120
110
  end
121
111
  end
122
112
 
123
113
  class Encoder
114
+ attr_reader :content_type
115
+
124
116
  def initialize(body, deflater)
117
+ @content_type = body.content_type
125
118
  @body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
126
119
  @buffer = StringIO.new("".b, File::RDWR)
127
120
  @deflater = deflater
@@ -130,11 +123,10 @@ module HTTPX
130
123
  def each(&blk)
131
124
  return enum_for(__method__) unless block_given?
132
125
 
133
- unless @buffer.size.zero?
134
- @buffer.rewind
135
- return @buffer.each(&blk)
136
- end
137
- deflate(&blk)
126
+ return deflate(&blk) if @buffer.size.zero?
127
+
128
+ @buffer.rewind
129
+ @buffer.each(&blk)
138
130
  end
139
131
 
140
132
  def bytesize
@@ -142,17 +134,6 @@ module HTTPX
142
134
  @buffer.size
143
135
  end
144
136
 
145
- def to_s
146
- deflate
147
- @buffer.rewind
148
- @buffer.read
149
- end
150
-
151
- def close
152
- @buffer.close
153
- @body.close
154
- end
155
-
156
137
  private
157
138
 
158
139
  def deflate(&blk)
@@ -162,22 +143,6 @@ module HTTPX
162
143
  @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
163
144
  end
164
145
  end
165
-
166
- class Decoder
167
- extend Forwardable
168
-
169
- def_delegator :@inflater, :finish
170
-
171
- def_delegator :@inflater, :close
172
-
173
- def initialize(inflater)
174
- @inflater = inflater
175
- end
176
-
177
- def decode(chunk)
178
- @inflater.inflate(chunk)
179
- end
180
- end
181
146
  end
182
147
  register_plugin :compression, Compression
183
148
  end
@@ -13,7 +13,7 @@ module HTTPX
13
13
  Compression.register "br", self
14
14
  end
15
15
 
16
- module Encoder
16
+ module Deflater
17
17
  module_function
18
18
 
19
19
  def deflate(raw, buffer, chunk_size:)
@@ -25,28 +25,24 @@ module HTTPX
25
25
  end
26
26
  end
27
27
 
28
- module BrotliWrapper
29
- module_function
30
-
31
- def inflate(text)
32
- ::Brotli.inflate(text)
28
+ class Inflater
29
+ def initialize(bytesize)
30
+ @bytesize = bytesize
33
31
  end
34
32
 
35
- def close; end
36
-
37
- def finish
38
- ""
33
+ def inflate(chunk)
34
+ ::Brotli.inflate(chunk)
39
35
  end
40
36
  end
41
37
 
42
38
  module_function
43
39
 
44
- def encoder
45
- Encoder
40
+ def deflater
41
+ Deflater
46
42
  end
47
43
 
48
- def decoder
49
- Decoder.new(BrotliWrapper)
44
+ def inflater(bytesize)
45
+ Inflater.new(bytesize)
50
46
  end
51
47
  end
52
48
  end