httpx 0.9.0 → 0.11.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +13 -3
  4. data/doc/release_notes/0_10_0.md +66 -0
  5. data/doc/release_notes/0_10_1.md +37 -0
  6. data/doc/release_notes/0_10_2.md +5 -0
  7. data/doc/release_notes/0_11_0.md +76 -0
  8. data/doc/release_notes/0_11_1.md +1 -0
  9. data/lib/httpx.rb +2 -0
  10. data/lib/httpx/adapters/datadog.rb +205 -0
  11. data/lib/httpx/adapters/faraday.rb +1 -3
  12. data/lib/httpx/adapters/webmock.rb +123 -0
  13. data/lib/httpx/chainable.rb +10 -9
  14. data/lib/httpx/connection.rb +7 -24
  15. data/lib/httpx/connection/http1.rb +15 -2
  16. data/lib/httpx/connection/http2.rb +15 -16
  17. data/lib/httpx/domain_name.rb +438 -0
  18. data/lib/httpx/errors.rb +4 -1
  19. data/lib/httpx/extensions.rb +21 -1
  20. data/lib/httpx/headers.rb +1 -0
  21. data/lib/httpx/io/ssl.rb +4 -9
  22. data/lib/httpx/io/tcp.rb +6 -5
  23. data/lib/httpx/io/udp.rb +8 -4
  24. data/lib/httpx/options.rb +2 -0
  25. data/lib/httpx/parser/http1.rb +14 -17
  26. data/lib/httpx/plugins/compression.rb +28 -63
  27. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  28. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  29. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  30. data/lib/httpx/plugins/cookies.rb +21 -60
  31. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  32. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  33. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  34. data/lib/httpx/plugins/expect.rb +34 -11
  35. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  36. data/lib/httpx/plugins/h2c.rb +1 -1
  37. data/lib/httpx/plugins/multipart.rb +41 -30
  38. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  39. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  40. data/lib/httpx/plugins/multipart/part.rb +34 -0
  41. data/lib/httpx/plugins/persistent.rb +6 -1
  42. data/lib/httpx/plugins/proxy.rb +16 -2
  43. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  44. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  45. data/lib/httpx/plugins/push_promise.rb +2 -2
  46. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  47. data/lib/httpx/plugins/retries.rb +3 -2
  48. data/lib/httpx/plugins/stream.rb +109 -13
  49. data/lib/httpx/pool.rb +14 -20
  50. data/lib/httpx/request.rb +29 -31
  51. data/lib/httpx/resolver.rb +7 -6
  52. data/lib/httpx/resolver/https.rb +25 -25
  53. data/lib/httpx/resolver/native.rb +29 -22
  54. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  55. data/lib/httpx/resolver/system.rb +3 -3
  56. data/lib/httpx/response.rb +16 -23
  57. data/lib/httpx/selector.rb +11 -17
  58. data/lib/httpx/session.rb +39 -30
  59. data/lib/httpx/transcoder.rb +20 -0
  60. data/lib/httpx/transcoder/chunker.rb +0 -2
  61. data/lib/httpx/transcoder/form.rb +9 -7
  62. data/lib/httpx/transcoder/json.rb +0 -4
  63. data/lib/httpx/utils.rb +45 -0
  64. data/lib/httpx/version.rb +1 -1
  65. data/sig/buffer.rbs +24 -0
  66. data/sig/callbacks.rbs +14 -0
  67. data/sig/chainable.rbs +37 -0
  68. data/sig/connection.rbs +85 -0
  69. data/sig/connection/http1.rbs +66 -0
  70. data/sig/connection/http2.rbs +77 -0
  71. data/sig/domain_name.rbs +17 -0
  72. data/sig/errors.rbs +3 -0
  73. data/sig/headers.rbs +45 -0
  74. data/sig/httpx.rbs +15 -0
  75. data/sig/loggable.rbs +11 -0
  76. data/sig/options.rbs +118 -0
  77. data/sig/parser/http1.rbs +50 -0
  78. data/sig/plugins/authentication.rbs +11 -0
  79. data/sig/plugins/basic_authentication.rbs +13 -0
  80. data/sig/plugins/compression.rbs +55 -0
  81. data/sig/plugins/compression/brotli.rbs +21 -0
  82. data/sig/plugins/compression/deflate.rbs +17 -0
  83. data/sig/plugins/compression/gzip.rbs +29 -0
  84. data/sig/plugins/cookies.rbs +26 -0
  85. data/sig/plugins/cookies/cookie.rbs +50 -0
  86. data/sig/plugins/cookies/jar.rbs +27 -0
  87. data/sig/plugins/digest_authentication.rbs +33 -0
  88. data/sig/plugins/expect.rbs +19 -0
  89. data/sig/plugins/follow_redirects.rbs +37 -0
  90. data/sig/plugins/h2c.rbs +26 -0
  91. data/sig/plugins/multipart.rbs +44 -0
  92. data/sig/plugins/persistent.rbs +17 -0
  93. data/sig/plugins/proxy.rbs +47 -0
  94. data/sig/plugins/proxy/http.rbs +14 -0
  95. data/sig/plugins/proxy/socks4.rbs +33 -0
  96. data/sig/plugins/proxy/socks5.rbs +36 -0
  97. data/sig/plugins/proxy/ssh.rbs +18 -0
  98. data/sig/plugins/push_promise.rbs +22 -0
  99. data/sig/plugins/rate_limiter.rbs +11 -0
  100. data/sig/plugins/retries.rbs +48 -0
  101. data/sig/plugins/stream.rbs +39 -0
  102. data/sig/pool.rbs +36 -0
  103. data/sig/registry.rbs +9 -0
  104. data/sig/request.rbs +61 -0
  105. data/sig/resolver.rbs +26 -0
  106. data/sig/resolver/https.rbs +51 -0
  107. data/sig/resolver/native.rbs +60 -0
  108. data/sig/resolver/resolver_mixin.rbs +27 -0
  109. data/sig/resolver/system.rbs +17 -0
  110. data/sig/response.rbs +87 -0
  111. data/sig/selector.rbs +20 -0
  112. data/sig/session.rbs +49 -0
  113. data/sig/timeout.rbs +29 -0
  114. data/sig/transcoder.rbs +18 -0
  115. data/sig/transcoder/body.rbs +20 -0
  116. data/sig/transcoder/chunker.rbs +32 -0
  117. data/sig/transcoder/form.rbs +22 -0
  118. data/sig/transcoder/json.rbs +16 -0
  119. metadata +99 -59
  120. data/lib/httpx/resolver/options.rb +0 -25
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Multipart
5
+ module MimeTypeDetector
6
+ module_function
7
+
8
+ DEFAULT_MIMETYPE = "application/octet-stream"
9
+
10
+ # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
11
+ if defined?(MIME::Types)
12
+
13
+ def call(_file, filename)
14
+ mime = MIME::Types.of(filename).first
15
+ mime.content_type if mime
16
+ end
17
+
18
+ elsif defined?(MimeMagic)
19
+
20
+ def call(file, *)
21
+ mime = MimeMagic.by_magic(file)
22
+ mime.type if mime
23
+ end
24
+
25
+ elsif system("which file", out: File::NULL)
26
+ require "open3"
27
+
28
+ def call(file, *)
29
+ return if file.eof? # file command returns "application/x-empty" for empty files
30
+
31
+ Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
32
+ begin
33
+ ::IO.copy_stream(file, stdin.binmode)
34
+ rescue Errno::EPIPE
35
+ end
36
+ file.rewind
37
+ stdin.close
38
+
39
+ status = thread.value
40
+
41
+ # call to file command failed
42
+ if status.nil? || !status.success?
43
+ $stderr.print(stderr.read)
44
+ else
45
+
46
+ output = stdout.read.strip
47
+
48
+ if output.include?("cannot open")
49
+ $stderr.print(output)
50
+ else
51
+ output
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ else
58
+
59
+ def call(*); end
60
+
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Multipart
5
+ module Part
6
+ module_function
7
+
8
+ def call(value)
9
+ # take out specialized objects of the way
10
+ if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
11
+ return [value, value.content_type, value.filename]
12
+ end
13
+
14
+ content_type = filename = nil
15
+
16
+ if value.is_a?(Hash)
17
+ content_type = value[:content_type]
18
+ filename = value[:filename]
19
+ value = value[:body]
20
+ end
21
+
22
+ value = value.open(:binmode => true) if value.is_a?(Pathname)
23
+
24
+ if value.is_a?(File)
25
+ filename ||= File.basename(value.path)
26
+ content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
27
+ [value, content_type, filename]
28
+ else
29
+ [StringIO.new(value.to_s), "text/plain"]
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -19,7 +19,12 @@ module HTTPX
19
19
  #
20
20
  module Persistent
21
21
  def self.load_dependencies(klass)
22
- klass.plugin(:retries, max_retries: 1, retry_change_requests: true)
22
+ max_retries = if klass.default_options.respond_to?(:max_retries)
23
+ [klass.default_options.max_retries, 1].max
24
+ else
25
+ 1
26
+ end
27
+ klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
23
28
  end
24
29
 
25
30
  def self.extra_options(options)
@@ -121,8 +121,7 @@ module HTTPX
121
121
  def fetch_response(request, connections, options)
122
122
  response = super
123
123
  if response.is_a?(ErrorResponse) &&
124
- # either it was a timeout error connecting, or it was a proxy error
125
- PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
124
+ __proxy_error?(response) && !@_proxy_uris.empty?
126
125
  @_proxy_uris.shift
127
126
  log { "failed connecting to proxy, trying next..." }
128
127
  request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
139
138
 
140
139
  super
141
140
  end
141
+
142
+ def __proxy_error?(response)
143
+ error = response.error
144
+ case error
145
+ when ResolveError
146
+ # failed resolving proxy domain
147
+ proxy_uri = error.connection.options.proxy.uri
148
+ proxy_uri.to_s == @_proxy_uris.first
149
+ when *PROXY_ERRORS
150
+ # timeout errors connecting to proxy
151
+ true
152
+ else
153
+ false
154
+ end
155
+ end
142
156
  end
143
157
 
144
158
  module ConnectionMethods
@@ -10,7 +10,7 @@ module HTTPX
10
10
  module Socks4
11
11
  VERSION = 4
12
12
  CONNECT = 1
13
- GRANTED = 90
13
+ GRANTED = 0x5A
14
14
  PROTOCOLS = %w[socks4 socks4a].freeze
15
15
 
16
16
  Error = Socks4Error
@@ -95,21 +95,21 @@ module HTTPX
95
95
 
96
96
  def connect(parameters, uri)
97
97
  packet = [VERSION, CONNECT, uri.port].pack("CCn")
98
- begin
99
- ip = IPAddr.new(uri.host)
100
- raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
101
-
102
- packet << [ip.to_i].pack("N")
103
- rescue IPAddr::InvalidAddressError
104
- if parameters.uri.scheme =~ /^socks4a?$/
105
- # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
106
- ip = IPAddr.new(Resolv.getaddress(uri.host))
107
- packet << [ip.to_i].pack("N")
108
- else
109
- packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
98
+
99
+ case parameters.uri.scheme
100
+ when "socks4"
101
+ socks_host = uri.host
102
+ begin
103
+ ip = IPAddr.new(socks_host)
104
+ packet << ip.hton
105
+ rescue IPAddr::InvalidAddressError
106
+ socks_host = Resolv.getaddress(socks_host)
107
+ retry
110
108
  end
109
+ packet << [parameters.username].pack("Z*")
110
+ when "socks4a"
111
+ packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
111
112
  end
112
- packet << [parameters.username].pack("Z*")
113
113
  packet
114
114
  end
115
115
  end
@@ -159,9 +159,10 @@ module HTTPX
159
159
  packet = [VERSION, CONNECT, 0].pack("C*")
160
160
  begin
161
161
  ip = IPAddr.new(uri.host)
162
- raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
163
162
 
164
- packet << [IPV4, ip.to_i].pack("CN")
163
+ ipcode = ip.ipv6? ? IPV6 : IPV4
164
+
165
+ packet << [ipcode].pack("C") << ip.hton
165
166
  rescue IPAddr::InvalidAddressError
166
167
  packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
167
168
  end
@@ -70,8 +70,8 @@ module HTTPX
70
70
  request.transition(:done)
71
71
  response = request.response
72
72
  response.mark_as_pushed!
73
- stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
74
- stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
73
+ stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
74
+ stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
75
75
  end
76
76
  end
77
77
  end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin adds support for retrying requests when the request:
7
+ #
8
+ # * is rate limited;
9
+ # * when the server is unavailable (503);
10
+ # * when a 3xx request comes with a "retry-after" value
11
+ #
12
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
13
+ #
14
+ module RateLimiter
15
+ class << self
16
+ RATE_LIMIT_CODES = [429, 503].freeze
17
+
18
+ def load_dependencies(klass)
19
+ klass.plugin(:retries,
20
+ retry_change_requests: true,
21
+ retry_on: method(:retry_on_rate_limited_response),
22
+ retry_after: method(:retry_after_rate_limit))
23
+ end
24
+
25
+ def retry_on_rate_limited_response(response)
26
+ status = response.status
27
+
28
+ RATE_LIMIT_CODES.include?(status)
29
+ end
30
+
31
+ # Servers send the "Retry-After" header field to indicate how long the
32
+ # user agent ought to wait before making a follow-up request. When
33
+ # sent with a 503 (Service Unavailable) response, Retry-After indicates
34
+ # how long the service is expected to be unavailable to the client.
35
+ # When sent with any 3xx (Redirection) response, Retry-After indicates
36
+ # the minimum time that the user agent is asked to wait before issuing
37
+ # the redirected request.
38
+ #
39
+ def retry_after_rate_limit(_, response)
40
+ retry_after = response.headers["retry-after"]
41
+
42
+ return unless retry_after
43
+
44
+ Utils.parse_retry_after(retry_after)
45
+ end
46
+ end
47
+ end
48
+
49
+ register_plugin :rate_limiter, RateLimiter
50
+ end
51
+ end
@@ -75,14 +75,15 @@ module HTTPX
75
75
  )
76
76
  # rubocop:enable Style/MultilineTernaryOperator
77
77
  )
78
-
78
+ response.close if response.respond_to?(:close)
79
79
  request.retries -= 1
80
80
  log { "failed to get response, #{request.retries} tries to go..." }
81
81
  request.transition(:idle)
82
82
 
83
83
  retry_after = options.retry_after
84
+ retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
85
+
84
86
  if retry_after
85
- retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
86
87
 
87
88
  log { "retrying after #{retry_after} secs..." }
88
89
  pool.after(retry_after) do
@@ -7,27 +7,123 @@ module HTTPX
7
7
  #
8
8
  module Stream
9
9
  module InstanceMethods
10
- def stream
11
- headers("accept" => "text/event-stream",
12
- "cache-control" => "no-cache")
10
+ private
11
+
12
+ def request(*args, stream: false, **options)
13
+ return super(*args, **options) unless stream
14
+
15
+ requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
16
+
17
+ raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
18
+
19
+ StreamResponse.new(requests.first, self)
13
20
  end
14
21
  end
15
22
 
23
+ module RequestMethods
24
+ attr_accessor :stream
25
+ end
26
+
16
27
  module ResponseMethods
17
- def complete?
18
- super ||
19
- stream? &&
20
- @stream_complete
28
+ def stream
29
+ @request.stream
30
+ end
31
+ end
32
+
33
+ module ResponseBodyMethods
34
+ def initialize(*, **)
35
+ super
36
+ @stream = @response.stream
37
+ end
38
+
39
+ def write(chunk)
40
+ return super unless @stream
41
+
42
+ @stream.on_chunk(chunk.to_s.dup)
43
+ end
44
+
45
+ private
46
+
47
+ def transition(*)
48
+ return if @stream
49
+
50
+ super
51
+ end
52
+ end
53
+
54
+ class StreamResponse
55
+ def initialize(request, session)
56
+ @request = request
57
+ @session = session
58
+ @options = @request.options
59
+ end
60
+
61
+ def each(&block)
62
+ return enum_for(__method__) unless block_given?
63
+
64
+ raise Error, "response already streamed" if @response
65
+
66
+ @request.stream = self
67
+
68
+ begin
69
+ @on_chunk = block
70
+
71
+ response.raise_for_status
72
+ response.close
73
+ ensure
74
+ @on_chunk = nil
75
+ end
76
+ end
77
+
78
+ def each_line
79
+ return enum_for(__method__) unless block_given?
80
+
81
+ line = +""
82
+
83
+ each do |chunk|
84
+ line << chunk
85
+
86
+ while (idx = line.index("\n"))
87
+ yield line.byteslice(0..idx - 1)
88
+
89
+ line = line.byteslice(idx + 1..-1)
90
+ end
91
+ end
92
+ end
93
+
94
+ # This is a ghost method. It's to be used ONLY internally, when processing streams
95
+ def on_chunk(chunk)
96
+ raise NoMethodError unless @on_chunk
97
+
98
+ @on_chunk.call(chunk)
99
+ end
100
+
101
+ # :nocov:
102
+ def inspect
103
+ "#<StreamResponse:#{object_id}>"
104
+ end
105
+ # :nocov:
106
+
107
+ def to_s
108
+ response.to_s
109
+ end
110
+
111
+ private
112
+
113
+ def response
114
+ @response ||= @session.__send__(:send_requests, @request, @options).first
21
115
  end
22
116
 
23
- def stream?
24
- @headers["content-type"].start_with?("text/event-stream")
117
+ def respond_to_missing?(*args)
118
+ @options.response_class.respond_to?(*args) || super
25
119
  end
26
120
 
27
- def <<(data)
28
- res = super
29
- @stream_complete = true if String(data).end_with?("\n\n")
30
- res
121
+ def method_missing(meth, *args, &block)
122
+ if @options.response_class.public_method_defined?(meth)
123
+ response.__send__(meth, *args, &block)
124
+ else
125
+ super
126
+ end
31
127
  end
32
128
  end
33
129
  end
@@ -37,20 +37,22 @@ module HTTPX
37
37
 
38
38
  @timers.fire
39
39
  end
40
- rescue Interrupt
41
- @connections.each(&:reset)
42
- raise
43
40
  rescue StandardError => e
44
41
  @connections.each do |connection|
45
42
  connection.emit(:error, e)
46
43
  end
44
+ rescue Exception # rubocop:disable Lint/RescueException
45
+ @connections.each(&:reset)
46
+ raise
47
47
  end
48
48
 
49
49
  def close(connections = @connections)
50
+ return if connections.empty?
51
+
50
52
  @timers.cancel
51
53
  connections = connections.reject(&:inflight?)
52
54
  connections.each(&:close)
53
- next_tick until connections.none? { |c| @connections.include?(c) }
55
+ next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
54
56
  @resolvers.each_value do |resolver|
55
57
  resolver.close unless resolver.closed?
56
58
  end if @connections.empty?
@@ -106,10 +108,10 @@ module HTTPX
106
108
  end
107
109
  end
108
110
 
109
- def on_resolver_error(ch, error)
110
- ch.emit(:error, error)
111
+ def on_resolver_error(connection, error)
112
+ connection.emit(:error, error)
111
113
  # must remove connection by hand, hasn't been started yet
112
- unregister_connection(ch)
114
+ unregister_connection(connection)
113
115
  end
114
116
 
115
117
  def on_resolver_close(resolver)
@@ -142,12 +144,12 @@ module HTTPX
142
144
  @connected_connections -= 1
143
145
  end
144
146
 
145
- def coalesce_connections(ch1, ch2)
146
- if ch1.coalescable?(ch2)
147
- ch1.merge(ch2)
148
- @connections.delete(ch2)
147
+ def coalesce_connections(conn1, conn2)
148
+ if conn1.coalescable?(conn2)
149
+ conn1.merge(conn2)
150
+ @connections.delete(conn2)
149
151
  else
150
- register_connection(ch2)
152
+ register_connection(conn2)
151
153
  end
152
154
  end
153
155
 
@@ -166,15 +168,7 @@ module HTTPX
166
168
  resolver.on(:error, &method(:on_resolver_error))
167
169
  resolver.on(:close) { on_resolver_close(resolver) }
168
170
  resolver
169
- # rubocop: disable Layout/RescueEnsureAlignment
170
- rescue ArgumentError
171
- # this block is here because of an error which happens on CI from time to time
172
- warn "tried resolver: #{resolver_type}"
173
- warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
174
- warn "new: #{resolver_type.method(:new).source_location}"
175
- raise
176
171
  end
177
- # rubocop: enable Layout/RescueEnsureAlignment
178
172
  end
179
173
  end
180
174
  end