httpx 0.10.0 → 0.11.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -3
  3. data/doc/release_notes/0_10_1.md +37 -0
  4. data/doc/release_notes/0_10_2.md +5 -0
  5. data/doc/release_notes/0_11_0.md +76 -0
  6. data/doc/release_notes/0_11_1.md +1 -0
  7. data/doc/release_notes/0_11_2.md +5 -0
  8. data/lib/httpx/adapters/datadog.rb +205 -0
  9. data/lib/httpx/adapters/faraday.rb +0 -2
  10. data/lib/httpx/adapters/webmock.rb +123 -0
  11. data/lib/httpx/chainable.rb +8 -7
  12. data/lib/httpx/connection.rb +4 -15
  13. data/lib/httpx/connection/http1.rb +14 -1
  14. data/lib/httpx/connection/http2.rb +15 -16
  15. data/lib/httpx/domain_name.rb +1 -3
  16. data/lib/httpx/errors.rb +3 -1
  17. data/lib/httpx/headers.rb +1 -0
  18. data/lib/httpx/io/ssl.rb +4 -8
  19. data/lib/httpx/io/udp.rb +4 -3
  20. data/lib/httpx/plugins/compression.rb +1 -1
  21. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  22. data/lib/httpx/plugins/expect.rb +33 -8
  23. data/lib/httpx/plugins/multipart.rb +42 -23
  24. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  25. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  26. data/lib/httpx/plugins/multipart/part.rb +34 -0
  27. data/lib/httpx/plugins/proxy.rb +16 -2
  28. data/lib/httpx/plugins/proxy/socks4.rb +14 -16
  29. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  30. data/lib/httpx/plugins/push_promise.rb +2 -2
  31. data/lib/httpx/pool.rb +8 -14
  32. data/lib/httpx/request.rb +22 -12
  33. data/lib/httpx/resolver.rb +7 -6
  34. data/lib/httpx/resolver/https.rb +18 -23
  35. data/lib/httpx/resolver/native.rb +22 -19
  36. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  37. data/lib/httpx/resolver/system.rb +3 -3
  38. data/lib/httpx/selector.rb +9 -13
  39. data/lib/httpx/session.rb +24 -21
  40. data/lib/httpx/transcoder.rb +20 -0
  41. data/lib/httpx/transcoder/form.rb +9 -1
  42. data/lib/httpx/version.rb +1 -1
  43. data/sig/connection.rbs +84 -1
  44. data/sig/connection/http1.rbs +66 -0
  45. data/sig/connection/http2.rbs +73 -0
  46. data/sig/headers.rbs +3 -0
  47. data/sig/httpx.rbs +1 -0
  48. data/sig/options.rbs +3 -3
  49. data/sig/plugins/basic_authentication.rbs +1 -1
  50. data/sig/plugins/compression.rbs +1 -1
  51. data/sig/plugins/compression/brotli.rbs +1 -1
  52. data/sig/plugins/compression/deflate.rbs +1 -1
  53. data/sig/plugins/compression/gzip.rbs +1 -1
  54. data/sig/plugins/h2c.rbs +1 -1
  55. data/sig/plugins/multipart.rbs +29 -4
  56. data/sig/plugins/persistent.rbs +1 -1
  57. data/sig/plugins/proxy.rbs +2 -2
  58. data/sig/plugins/proxy/ssh.rbs +1 -1
  59. data/sig/plugins/rate_limiter.rbs +1 -1
  60. data/sig/pool.rbs +36 -2
  61. data/sig/request.rbs +2 -2
  62. data/sig/resolver.rbs +26 -0
  63. data/sig/resolver/https.rbs +51 -0
  64. data/sig/resolver/native.rbs +60 -0
  65. data/sig/resolver/resolver_mixin.rbs +27 -0
  66. data/sig/resolver/system.rbs +17 -0
  67. data/sig/response.rbs +2 -2
  68. data/sig/selector.rbs +20 -0
  69. data/sig/session.rbs +3 -3
  70. data/sig/transcoder.rbs +4 -2
  71. data/sig/transcoder/body.rbs +2 -0
  72. data/sig/transcoder/form.rbs +8 -2
  73. data/sig/transcoder/json.rbs +3 -1
  74. metadata +47 -48
  75. data/lib/httpx/resolver/options.rb +0 -25
  76. data/sig/missing.rbs +0 -12
  77. data/sig/test.rbs +0 -9
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX::Plugins
4
+ module Multipart
5
+ class Encoder
6
+ attr_reader :bytesize
7
+
8
+ def initialize(form)
9
+ @boundary = ("-" * 21) << SecureRandom.hex(21)
10
+ @part_index = 0
11
+ @buffer = "".b
12
+
13
+ @form = form
14
+ @parts = to_parts(form)
15
+ end
16
+
17
+ def content_type
18
+ "multipart/form-data; boundary=#{@boundary}"
19
+ end
20
+
21
+ def read(length = nil, outbuf = nil)
22
+ data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
23
+ data ||= "".b
24
+
25
+ read_chunks(data, length)
26
+
27
+ data unless length && data.empty?
28
+ end
29
+
30
+ def rewind
31
+ form = @form.each_with_object([]) do |(key, val), aux|
32
+ v = case val
33
+ when File
34
+ val = val.reopen(val.path, File::RDONLY) if val.closed?
35
+ val.rewind
36
+ val
37
+ else
38
+ v
39
+ end
40
+ aux << [key, v]
41
+ end
42
+ @form = form
43
+ @parts = to_parts(form)
44
+ end
45
+
46
+ private
47
+
48
+ def to_parts(form)
49
+ @bytesize = 0
50
+ params = form.each_with_object([]) do |(key, val), aux|
51
+ Multipart.normalize_keys(key, val) do |k, v|
52
+ next if v.nil?
53
+
54
+ value, content_type, filename = Part.call(v)
55
+
56
+ header = header_part(k, content_type, filename)
57
+ @bytesize += header.size
58
+ aux << header
59
+
60
+ @bytesize += value.size
61
+ aux << value
62
+
63
+ delimiter = StringIO.new("\r\n")
64
+ @bytesize += delimiter.size
65
+ aux << delimiter
66
+ end
67
+ end
68
+ final_delimiter = StringIO.new("--#{@boundary}--\r\n")
69
+ @bytesize += final_delimiter.size
70
+ params << final_delimiter
71
+
72
+ params
73
+ end
74
+
75
+ def header_part(key, content_type, filename)
76
+ header = "--#{@boundary}\r\n".b
77
+ header << "Content-Disposition: form-data; name=#{key.inspect}".b
78
+ header << "; filename=#{filename.inspect}" if filename
79
+ header << "\r\nContent-Type: #{content_type}\r\n\r\n"
80
+ StringIO.new(header)
81
+ end
82
+
83
+ def read_chunks(buffer, length = nil)
84
+ while @part_index < @parts.size
85
+ chunk = read_from_part(length)
86
+
87
+ next unless chunk
88
+
89
+ buffer << chunk.force_encoding(Encoding::BINARY)
90
+
91
+ next unless length
92
+
93
+ length -= chunk.bytesize
94
+
95
+ break if length.zero?
96
+ end
97
+ end
98
+
99
+ # if there's a current part to read from, tries to read a chunk.
100
+ def read_from_part(max_length = nil)
101
+ part = @parts[@part_index]
102
+
103
+ chunk = part.read(max_length, @buffer)
104
+
105
+ return chunk if chunk && !chunk.empty?
106
+
107
+ part.close if part.respond_to?(:close)
108
+
109
+ @part_index += 1
110
+
111
+ nil
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Multipart
5
+ module MimeTypeDetector
6
+ module_function
7
+
8
+ DEFAULT_MIMETYPE = "application/octet-stream"
9
+
10
+ # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
11
+ if defined?(MIME::Types)
12
+
13
+ def call(_file, filename)
14
+ mime = MIME::Types.of(filename).first
15
+ mime.content_type if mime
16
+ end
17
+
18
+ elsif defined?(MimeMagic)
19
+
20
+ def call(file, *)
21
+ mime = MimeMagic.by_magic(file)
22
+ mime.type if mime
23
+ end
24
+
25
+ elsif system("which file", out: File::NULL)
26
+ require "open3"
27
+
28
+ def call(file, *)
29
+ return if file.eof? # file command returns "application/x-empty" for empty files
30
+
31
+ Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
32
+ begin
33
+ ::IO.copy_stream(file, stdin.binmode)
34
+ rescue Errno::EPIPE
35
+ end
36
+ file.rewind
37
+ stdin.close
38
+
39
+ status = thread.value
40
+
41
+ # call to file command failed
42
+ if status.nil? || !status.success?
43
+ $stderr.print(stderr.read)
44
+ else
45
+
46
+ output = stdout.read.strip
47
+
48
+ if output.include?("cannot open")
49
+ $stderr.print(output)
50
+ else
51
+ output
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ else
58
+
59
+ def call(*); end
60
+
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Multipart
5
+ module Part
6
+ module_function
7
+
8
+ def call(value)
9
+ # take out specialized objects of the way
10
+ if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
11
+ return [value, value.content_type, value.filename]
12
+ end
13
+
14
+ content_type = filename = nil
15
+
16
+ if value.is_a?(Hash)
17
+ content_type = value[:content_type]
18
+ filename = value[:filename]
19
+ value = value[:body]
20
+ end
21
+
22
+ value = value.open(:binmode => true) if value.is_a?(Pathname)
23
+
24
+ if value.is_a?(File)
25
+ filename ||= File.basename(value.path)
26
+ content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
27
+ [value, content_type, filename]
28
+ else
29
+ [StringIO.new(value.to_s), "text/plain"]
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -121,8 +121,7 @@ module HTTPX
121
121
  def fetch_response(request, connections, options)
122
122
  response = super
123
123
  if response.is_a?(ErrorResponse) &&
124
- # either it was a timeout error connecting, or it was a proxy error
125
- PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
124
+ __proxy_error?(response) && !@_proxy_uris.empty?
126
125
  @_proxy_uris.shift
127
126
  log { "failed connecting to proxy, trying next..." }
128
127
  request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
139
138
 
140
139
  super
141
140
  end
141
+
142
+ def __proxy_error?(response)
143
+ error = response.error
144
+ case error
145
+ when ResolveError
146
+ # failed resolving proxy domain
147
+ proxy_uri = error.connection.options.proxy.uri
148
+ proxy_uri.to_s == @_proxy_uris.first
149
+ when *PROXY_ERRORS
150
+ # timeout errors connecting to proxy
151
+ true
152
+ else
153
+ false
154
+ end
155
+ end
142
156
  end
143
157
 
144
158
  module ConnectionMethods
@@ -10,7 +10,7 @@ module HTTPX
10
10
  module Socks4
11
11
  VERSION = 4
12
12
  CONNECT = 1
13
- GRANTED = 90
13
+ GRANTED = 0x5A
14
14
  PROTOCOLS = %w[socks4 socks4a].freeze
15
15
 
16
16
  Error = Socks4Error
@@ -91,27 +91,25 @@ module HTTPX
91
91
  end
92
92
 
93
93
  module Packet
94
- using(RegexpExtensions) unless Regexp.method_defined?(:match?)
95
-
96
94
  module_function
97
95
 
98
96
  def connect(parameters, uri)
99
97
  packet = [VERSION, CONNECT, uri.port].pack("CCn")
100
- begin
101
- ip = IPAddr.new(uri.host)
102
- raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
103
-
104
- packet << [ip.to_i].pack("N")
105
- rescue IPAddr::InvalidAddressError
106
- if /^socks4a?$/.match?(parameters.uri.scheme)
107
- # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
108
- ip = IPAddr.new(Resolv.getaddress(uri.host))
109
- packet << [ip.to_i].pack("N")
110
- else
111
- packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
98
+
99
+ case parameters.uri.scheme
100
+ when "socks4"
101
+ socks_host = uri.host
102
+ begin
103
+ ip = IPAddr.new(socks_host)
104
+ packet << ip.hton
105
+ rescue IPAddr::InvalidAddressError
106
+ socks_host = Resolv.getaddress(socks_host)
107
+ retry
112
108
  end
109
+ packet << [parameters.username].pack("Z*")
110
+ when "socks4a"
111
+ packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
113
112
  end
114
- packet << [parameters.username].pack("Z*")
115
113
  packet
116
114
  end
117
115
  end
@@ -159,9 +159,10 @@ module HTTPX
159
159
  packet = [VERSION, CONNECT, 0].pack("C*")
160
160
  begin
161
161
  ip = IPAddr.new(uri.host)
162
- raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
163
162
 
164
- packet << [IPV4, ip.to_i].pack("CN")
163
+ ipcode = ip.ipv6? ? IPV6 : IPV4
164
+
165
+ packet << [ipcode].pack("C") << ip.hton
165
166
  rescue IPAddr::InvalidAddressError
166
167
  packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
167
168
  end
@@ -70,8 +70,8 @@ module HTTPX
70
70
  request.transition(:done)
71
71
  response = request.response
72
72
  response.mark_as_pushed!
73
- stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
74
- stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
73
+ stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
74
+ stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
75
75
  end
76
76
  end
77
77
  end
data/lib/httpx/pool.rb CHANGED
@@ -108,10 +108,10 @@ module HTTPX
108
108
  end
109
109
  end
110
110
 
111
- def on_resolver_error(ch, error)
112
- ch.emit(:error, error)
111
+ def on_resolver_error(connection, error)
112
+ connection.emit(:error, error)
113
113
  # must remove connection by hand, hasn't been started yet
114
- unregister_connection(ch)
114
+ unregister_connection(connection)
115
115
  end
116
116
 
117
117
  def on_resolver_close(resolver)
@@ -144,12 +144,12 @@ module HTTPX
144
144
  @connected_connections -= 1
145
145
  end
146
146
 
147
- def coalesce_connections(ch1, ch2)
148
- if ch1.coalescable?(ch2)
149
- ch1.merge(ch2)
150
- @connections.delete(ch2)
147
+ def coalesce_connections(conn1, conn2)
148
+ if conn1.coalescable?(conn2)
149
+ conn1.merge(conn2)
150
+ @connections.delete(conn2)
151
151
  else
152
- register_connection(ch2)
152
+ register_connection(conn2)
153
153
  end
154
154
  end
155
155
 
@@ -168,12 +168,6 @@ module HTTPX
168
168
  resolver.on(:error, &method(:on_resolver_error))
169
169
  resolver.on(:close) { on_resolver_close(resolver) }
170
170
  resolver
171
- rescue ArgumentError
172
- # this block is here because of an error which happens on CI from time to time
173
- warn "tried resolver: #{resolver_type}"
174
- warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
175
- warn "new: #{resolver_type.method(:new).source_location}"
176
- raise
177
171
  end
178
172
  end
179
173
  end
data/lib/httpx/request.rb CHANGED
@@ -81,6 +81,10 @@ module HTTPX
81
81
  def response=(response)
82
82
  return unless response
83
83
 
84
+ if response.status == 100
85
+ @informational_status = response.status
86
+ return
87
+ end
84
88
  @response = response
85
89
  end
86
90
 
@@ -106,7 +110,7 @@ module HTTPX
106
110
 
107
111
  query = []
108
112
  if (q = @options.params)
109
- query << URI.encode_www_form(q)
113
+ query << Transcoder.registry("form").encode(q)
110
114
  end
111
115
  query << @uri.query if @uri.query
112
116
  @query = query.join("&")
@@ -170,6 +174,12 @@ module HTTPX
170
174
  end
171
175
  end
172
176
 
177
+ def rewind
178
+ return if empty?
179
+
180
+ @body.rewind if @body.respond_to?(:rewind)
181
+ end
182
+
173
183
  def empty?
174
184
  return true if @body.nil?
175
185
  return false if chunked?
@@ -212,6 +222,7 @@ module HTTPX
212
222
  def transition(nextstate)
213
223
  case nextstate
214
224
  when :idle
225
+ @body.rewind
215
226
  @response = nil
216
227
  @drainer = nil
217
228
  when :headers
@@ -221,15 +232,15 @@ module HTTPX
221
232
  @state == :expect
222
233
 
223
234
  if @headers.key?("expect")
224
- unless @response
225
- @state = :expect
226
- return
227
- end
228
-
229
- case @response.status
230
- when 100
231
- # deallocate
232
- @response = nil
235
+ if @informational_status && @informational_status == 100
236
+ # check for 100 Continue response, and deallocate the var
237
+ # if @informational_status == 100
238
+ # @response = nil
239
+ # end
240
+ else
241
+ return if @state == :expect # do not re-set it
242
+
243
+ nextstate = :expect
233
244
  end
234
245
  end
235
246
  when :done
@@ -241,8 +252,7 @@ module HTTPX
241
252
  end
242
253
 
243
254
  def expects?
244
- @headers["expect"] == "100-continue" &&
245
- @response && @response.status == 100
255
+ @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
246
256
  end
247
257
 
248
258
  class ProcIO
@@ -1,15 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "resolv"
4
- require "httpx/resolver/resolver_mixin"
5
- require "httpx/resolver/system"
6
- require "httpx/resolver/native"
7
- require "httpx/resolver/https"
8
4
 
9
5
  module HTTPX
10
6
  module Resolver
11
7
  extend Registry
12
8
 
9
+ RESOLVE_TIMEOUT = 5
10
+
11
+ require "httpx/resolver/resolver_mixin"
12
+ require "httpx/resolver/system"
13
+ require "httpx/resolver/native"
14
+ require "httpx/resolver/https"
15
+
13
16
  register :system, System
14
17
  register :native, Native
15
18
  register :https, HTTPS
@@ -101,5 +104,3 @@ module HTTPX
101
104
  end
102
105
  end
103
106
  end
104
-
105
- require "httpx/resolver/options"