httpx 0.12.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/doc/release_notes/0_10_1.md +1 -1
  3. data/doc/release_notes/0_13_0.md +58 -0
  4. data/doc/release_notes/0_13_1.md +5 -0
  5. data/doc/release_notes/0_13_2.md +9 -0
  6. data/doc/release_notes/0_14_0.md +79 -0
  7. data/doc/release_notes/0_14_1.md +7 -0
  8. data/lib/httpx.rb +1 -2
  9. data/lib/httpx/callbacks.rb +12 -3
  10. data/lib/httpx/chainable.rb +2 -2
  11. data/lib/httpx/connection.rb +29 -22
  12. data/lib/httpx/connection/http1.rb +35 -15
  13. data/lib/httpx/connection/http2.rb +61 -15
  14. data/lib/httpx/headers.rb +7 -3
  15. data/lib/httpx/io/ssl.rb +30 -17
  16. data/lib/httpx/io/tcp.rb +48 -27
  17. data/lib/httpx/io/udp.rb +31 -7
  18. data/lib/httpx/io/unix.rb +27 -12
  19. data/lib/httpx/options.rb +97 -74
  20. data/lib/httpx/plugins/aws_sdk_authentication.rb +5 -2
  21. data/lib/httpx/plugins/aws_sigv4.rb +5 -4
  22. data/lib/httpx/plugins/basic_authentication.rb +8 -3
  23. data/lib/httpx/plugins/compression.rb +24 -12
  24. data/lib/httpx/plugins/compression/brotli.rb +10 -7
  25. data/lib/httpx/plugins/compression/deflate.rb +6 -5
  26. data/lib/httpx/plugins/compression/gzip.rb +4 -3
  27. data/lib/httpx/plugins/cookies.rb +3 -7
  28. data/lib/httpx/plugins/digest_authentication.rb +5 -5
  29. data/lib/httpx/plugins/expect.rb +6 -6
  30. data/lib/httpx/plugins/follow_redirects.rb +4 -4
  31. data/lib/httpx/plugins/grpc.rb +247 -0
  32. data/lib/httpx/plugins/grpc/call.rb +62 -0
  33. data/lib/httpx/plugins/grpc/message.rb +85 -0
  34. data/lib/httpx/plugins/h2c.rb +43 -58
  35. data/lib/httpx/plugins/internal_telemetry.rb +1 -1
  36. data/lib/httpx/plugins/multipart/part.rb +2 -2
  37. data/lib/httpx/plugins/proxy.rb +3 -7
  38. data/lib/httpx/plugins/proxy/http.rb +5 -4
  39. data/lib/httpx/plugins/proxy/ssh.rb +3 -3
  40. data/lib/httpx/plugins/rate_limiter.rb +1 -1
  41. data/lib/httpx/plugins/retries.rb +14 -15
  42. data/lib/httpx/plugins/stream.rb +99 -75
  43. data/lib/httpx/plugins/upgrade.rb +84 -0
  44. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  45. data/lib/httpx/pool.rb +14 -5
  46. data/lib/httpx/request.rb +25 -2
  47. data/lib/httpx/resolver/native.rb +7 -3
  48. data/lib/httpx/response.rb +9 -5
  49. data/lib/httpx/session.rb +17 -7
  50. data/lib/httpx/transcoder/chunker.rb +1 -1
  51. data/lib/httpx/version.rb +1 -1
  52. data/sig/callbacks.rbs +2 -0
  53. data/sig/chainable.rbs +2 -1
  54. data/sig/connection/http1.rbs +6 -1
  55. data/sig/connection/http2.rbs +6 -2
  56. data/sig/headers.rbs +2 -2
  57. data/sig/options.rbs +16 -22
  58. data/sig/plugins/aws_sdk_authentication.rbs +2 -0
  59. data/sig/plugins/aws_sigv4.rbs +0 -1
  60. data/sig/plugins/basic_authentication.rbs +2 -0
  61. data/sig/plugins/compression.rbs +7 -5
  62. data/sig/plugins/compression/brotli.rbs +1 -1
  63. data/sig/plugins/compression/deflate.rbs +1 -1
  64. data/sig/plugins/compression/gzip.rbs +1 -1
  65. data/sig/plugins/cookies.rbs +0 -1
  66. data/sig/plugins/digest_authentication.rbs +0 -1
  67. data/sig/plugins/expect.rbs +0 -2
  68. data/sig/plugins/follow_redirects.rbs +0 -2
  69. data/sig/plugins/h2c.rbs +5 -10
  70. data/sig/plugins/persistent.rbs +0 -1
  71. data/sig/plugins/proxy.rbs +0 -1
  72. data/sig/plugins/retries.rbs +0 -4
  73. data/sig/plugins/stream.rbs +17 -16
  74. data/sig/plugins/upgrade.rbs +23 -0
  75. data/sig/request.rbs +7 -2
  76. data/sig/response.rbs +4 -1
  77. data/sig/session.rbs +4 -0
  78. metadata +21 -7
  79. data/lib/httpx/timeout.rb +0 -67
  80. data/sig/timeout.rbs +0 -29
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ module GRPC
6
+ # Encoding module for GRPC responses
7
+ #
8
+ # Can encode and decode grpc messages.
9
+ module Message
10
+ module_function
11
+
12
+ # decodes a unary grpc response
13
+ def unary(response)
14
+ verify_status(response)
15
+ decode(response.to_s, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders)
16
+ end
17
+
18
+ # lazy decodes a grpc stream response
19
+ def stream(response, &block)
20
+ return enum_for(__method__, response) unless block_given?
21
+
22
+ response.each do |frame|
23
+ decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
24
+ end
25
+
26
+ verify_status(response)
27
+ end
28
+
29
+ # encodes a single grpc message
30
+ def encode(bytes, deflater:)
31
+ if deflater
32
+ compressed_flag = 1
33
+ bytes = deflater.deflate(StringIO.new(bytes))
34
+ else
35
+ compressed_flag = 0
36
+ end
37
+
38
+ "".b << [compressed_flag, bytes.bytesize].pack("CL>") << bytes.to_s
39
+ end
40
+
41
+ # decodes a single grpc message
42
+ def decode(message, encodings:, encoders:)
43
+ until message.empty?
44
+
45
+ compressed, size = message.unpack("CL>")
46
+
47
+ data = message.byteslice(5..size + 5 - 1)
48
+ if compressed == 1
49
+ encodings.reverse_each do |algo|
50
+ inflater = encoders.registry(algo).inflater(size)
51
+ data = inflater.inflate(data)
52
+ size = data.bytesize
53
+ end
54
+ end
55
+
56
+ return data unless block_given?
57
+
58
+ yield data
59
+
60
+ message = message.byteslice(5 + size..-1)
61
+ end
62
+ end
63
+
64
+ def cancel(request)
65
+ request.emit(:refuse, :client_cancellation)
66
+ end
67
+
68
+ # interprets the grpc call trailing metadata, and raises an
69
+ # exception in case of error code
70
+ def verify_status(response)
71
+ # return standard errors if need be
72
+ response.raise_for_status
73
+
74
+ status = Integer(response.headers["grpc-status"])
75
+ message = response.headers["grpc-message"]
76
+
77
+ return if status.zero?
78
+
79
+ response.close
80
+ raise GRPCError.new(status, message, response.trailing_metadata)
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -6,68 +6,53 @@ module HTTPX
6
6
  # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
7
7
  # (https://tools.ietf.org/html/rfc7540#section-3.2)
8
8
  #
9
- # https://gitlab.com/honeyryderchuck/httpx/wikis/Follow-Redirects
9
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2c
10
10
  #
11
11
  module H2C
12
- def self.load_dependencies(*)
13
- require "base64"
12
+ VALID_H2C_VERBS = %i[get options head].freeze
13
+
14
+ class << self
15
+ def load_dependencies(*)
16
+ require "base64"
17
+ end
18
+
19
+ def configure(klass)
20
+ klass.plugin(:upgrade)
21
+ klass.default_options.upgrade_handlers.register "h2c", self
22
+ end
23
+
24
+ def call(connection, request, response)
25
+ connection.upgrade_to_h2c(request, response)
26
+ end
14
27
  end
15
28
 
16
29
  module InstanceMethods
17
- def request(*args, **options)
18
- h2c_options = options.merge(fallback_protocol: "h2c")
30
+ def send_requests(*requests, options)
31
+ upgrade_request, *remainder = requests
32
+
33
+ return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
19
34
 
20
- requests = build_requests(*args, h2c_options)
35
+ connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
21
36
 
22
- upgrade_request = requests.first
23
- return super unless valid_h2c_upgrade_request?(upgrade_request)
37
+ return super if connection && connection.upgrade_protocol == :h2c
24
38
 
39
+ # build upgrade request
25
40
  upgrade_request.headers.add("connection", "upgrade")
26
41
  upgrade_request.headers.add("connection", "http2-settings")
27
42
  upgrade_request.headers["upgrade"] = "h2c"
28
43
  upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
29
- wrap { send_requests(*upgrade_request, h2c_options).first }
30
44
 
31
- responses = send_requests(*requests, h2c_options)
32
-
33
- responses.size == 1 ? responses.first : responses
34
- end
35
-
36
- private
37
-
38
- def fetch_response(request, connections, options)
39
- response = super
40
- if response && valid_h2c_upgrade?(request, response, options)
41
- log { "upgrading to h2c..." }
42
- connection = find_connection(request, connections, options)
43
- connections << connection unless connections.include?(connection)
44
- connection.upgrade(request, response)
45
- end
46
- response
47
- end
48
-
49
- VALID_H2C_METHODS = %i[get options head].freeze
50
- private_constant :VALID_H2C_METHODS
51
-
52
- def valid_h2c_upgrade_request?(request)
53
- VALID_H2C_METHODS.include?(request.verb) &&
54
- request.scheme == "http"
55
- end
56
-
57
- def valid_h2c_upgrade?(request, response, options)
58
- options.fallback_protocol == "h2c" &&
59
- request.headers.get("connection").include?("upgrade") &&
60
- request.headers.get("upgrade").include?("h2c") &&
61
- response.status == 101
45
+ super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
62
46
  end
63
47
  end
64
48
 
65
49
  class H2CParser < Connection::HTTP2
66
50
  def upgrade(request, response)
67
- @connection.send_connection_preface
68
51
  # skip checks, it is assumed that this is the first
69
52
  # request in the connection
70
53
  stream = @connection.upgrade
54
+
55
+ # on_settings
71
56
  handle_stream(stream, request)
72
57
  @streams[request] = stream
73
58
 
@@ -81,29 +66,29 @@ module HTTPX
81
66
  module ConnectionMethods
82
67
  using URIExtensions
83
68
 
84
- def match?(uri, options)
85
- return super unless uri.scheme == "http" && @options.fallback_protocol == "h2c"
86
-
87
- super && options.fallback_protocol == "h2c"
88
- end
89
-
90
- def coalescable?(connection)
91
- return super unless @options.fallback_protocol == "h2c" && @origin.scheme == "http"
69
+ def upgrade_to_h2c(request, response)
70
+ prev_parser = @parser
92
71
 
93
- @origin == connection.origin && connection.options.fallback_protocol == "h2c"
94
- end
72
+ if prev_parser
73
+ prev_parser.reset
74
+ @inflight -= prev_parser.requests.size
75
+ end
95
76
 
96
- def upgrade(request, response)
97
- @parser.reset if @parser
98
- @parser = H2CParser.new(@write_buffer, @options)
77
+ parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
78
+ @parser = H2CParser.new(@write_buffer, parser_options)
99
79
  set_parser_callbacks(@parser)
80
+ @inflight += 1
100
81
  @parser.upgrade(request, response)
101
- end
82
+ @upgrade_protocol = :h2c
102
83
 
103
- def build_parser(*)
104
- return super unless @origin.scheme == "http"
84
+ if request.options.max_concurrent_requests != @options.max_concurrent_requests
85
+ @options = @options.merge(max_concurrent_requests: nil)
86
+ end
105
87
 
106
- super("http/1.1")
88
+ prev_parser.requests.each do |req|
89
+ req.transition(:idle)
90
+ send(req)
91
+ end
107
92
  end
108
93
  end
109
94
  end
@@ -71,7 +71,7 @@ module HTTPX
71
71
  def transition(nextstate)
72
72
  state = @state
73
73
  super
74
- meter_elapsed_time("Request[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
74
+ meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
75
75
  end
76
76
  end
77
77
 
@@ -19,14 +19,14 @@ module HTTPX
19
19
  value = value[:body]
20
20
  end
21
21
 
22
- value = value.open(:binmode => true) if value.is_a?(Pathname)
22
+ value = value.open(:binmode => true) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
23
23
 
24
24
  if value.is_a?(File)
25
25
  filename ||= File.basename(value.path)
26
26
  content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
27
27
  [value, content_type, filename]
28
28
  else
29
- [StringIO.new(value.to_s), "text/plain"]
29
+ [StringIO.new(value.to_s), content_type || "text/plain"]
30
30
  end
31
31
  end
32
32
  end
@@ -67,13 +67,9 @@ module HTTPX
67
67
 
68
68
  def extra_options(options)
69
69
  Class.new(options.class) do
70
- def_option(:proxy) do |pr|
71
- if pr.is_a?(Parameters)
72
- pr
73
- else
74
- Hash[pr]
75
- end
76
- end
70
+ def_option(:proxy, <<-OUT)
71
+ value.is_a?(#{Parameters}) ? value : Hash[value]
72
+ OUT
77
73
  end.new(options)
78
74
  end
79
75
  end
@@ -82,11 +82,12 @@ module HTTPX
82
82
  end
83
83
 
84
84
  def set_protocol_headers(request)
85
- super
85
+ extra_headers = super
86
+
86
87
  proxy_params = @options.proxy
87
- request.headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
88
- request.headers["proxy-connection"] = request.headers["connection"]
89
- request.headers.delete("connection")
88
+ extra_headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
89
+ extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
90
+ extra_headers
90
91
  end
91
92
  end
92
93
 
@@ -12,9 +12,9 @@ module HTTPX
12
12
 
13
13
  def self.extra_options(options)
14
14
  Class.new(options.class) do
15
- def_option(:proxy) do |pr|
16
- Hash[pr]
17
- end
15
+ def_option(:proxy, <<-OUT)
16
+ Hash[value]
17
+ OUT
18
18
  end.new(options)
19
19
  end
20
20
 
@@ -15,7 +15,7 @@ module HTTPX
15
15
  class << self
16
16
  RATE_LIMIT_CODES = [429, 503].freeze
17
17
 
18
- def load_dependencies(klass)
18
+ def configure(klass)
19
19
  klass.plugin(:retries,
20
20
  retry_change_requests: true,
21
21
  retry_on: method(:retry_on_rate_limited_response),
@@ -25,37 +25,36 @@ module HTTPX
25
25
 
26
26
  def self.extra_options(options)
27
27
  Class.new(options.class) do
28
- # number of seconds after which one can retry the request
29
- def_option(:retry_after) do |num|
28
+ def_option(:retry_after, <<-OUT)
30
29
  # return early if callable
31
- unless num.respond_to?(:call)
32
- num = Integer(num)
33
- raise Error, ":retry_after must be positive" unless num.positive?
30
+ unless value.respond_to?(:call)
31
+ value = Integer(value)
32
+ raise Error, ":retry_after must be positive" unless value.positive?
34
33
  end
35
34
 
36
- num
37
- end
35
+ value
36
+ OUT
38
37
 
39
- def_option(:max_retries) do |num|
40
- num = Integer(num)
38
+ def_option(:max_retries, <<-OUT)
39
+ num = Integer(value)
41
40
  raise Error, ":max_retries must be positive" unless num.positive?
42
41
 
43
42
  num
44
- end
43
+ OUT
45
44
 
46
45
  def_option(:retry_change_requests)
47
46
 
48
- def_option(:retry_on) do |callback|
49
- raise ":retry_on must be called with the response" unless callback.respond_to?(:call)
47
+ def_option(:retry_on, <<-OUT)
48
+ raise ":retry_on must be called with the response" unless value.respond_to?(:call)
50
49
 
51
- callback
52
- end
50
+ value
51
+ OUT
53
52
  end.new(options).merge(max_retries: MAX_RETRIES)
54
53
  end
55
54
 
56
55
  module InstanceMethods
57
56
  def max_retries(n)
58
- branch(default_options.with_max_retries(n.to_i))
57
+ with(max_retries: n.to_i)
59
58
  end
60
59
 
61
60
  private
@@ -1,10 +1,99 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
+ class StreamResponse
5
+ def initialize(request, session, connections)
6
+ @request = request
7
+ @session = session
8
+ @connections = connections
9
+ @options = @request.options
10
+ end
11
+
12
+ def each(&block)
13
+ return enum_for(__method__) unless block_given?
14
+
15
+ raise Error, "response already streamed" if @response
16
+
17
+ @request.stream = self
18
+
19
+ begin
20
+ @on_chunk = block
21
+
22
+ if @request.response
23
+ # if we've already started collecting the payload, yield it first
24
+ # before proceeding
25
+ body = @request.response.body
26
+
27
+ body.each do |chunk|
28
+ on_chunk(chunk)
29
+ end
30
+ end
31
+
32
+ response.raise_for_status
33
+ response.close
34
+ ensure
35
+ @on_chunk = nil
36
+ end
37
+ end
38
+
39
+ def each_line
40
+ return enum_for(__method__) unless block_given?
41
+
42
+ line = +""
43
+
44
+ each do |chunk|
45
+ line << chunk
46
+
47
+ while (idx = line.index("\n"))
48
+ yield line.byteslice(0..idx - 1)
49
+
50
+ line = line.byteslice(idx + 1..-1)
51
+ end
52
+ end
53
+ end
54
+
55
+ # This is a ghost method. It's to be used ONLY internally, when processing streams
56
+ def on_chunk(chunk)
57
+ raise NoMethodError unless @on_chunk
58
+
59
+ @on_chunk.call(chunk)
60
+ end
61
+
62
+ # :nocov:
63
+ def inspect
64
+ "#<StreamResponse:#{object_id}>"
65
+ end
66
+ # :nocov:
67
+
68
+ def to_s
69
+ response.to_s
70
+ end
71
+
72
+ private
73
+
74
+ def response
75
+ @session.__send__(:receive_requests, [@request], @connections, @options) until @request.response
76
+
77
+ @request.response
78
+ end
79
+
80
+ def respond_to_missing?(meth, *args)
81
+ response.respond_to?(meth, *args) || super
82
+ end
83
+
84
+ def method_missing(meth, *args, &block)
85
+ return super unless response.respond_to?(meth)
86
+
87
+ response.__send__(meth, *args, &block)
88
+ end
89
+ end
90
+
4
91
  module Plugins
5
92
  #
6
93
  # This plugin adds support for stream response (text/event-stream).
7
94
  #
95
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Stream
96
+ #
8
97
  module Stream
9
98
  module InstanceMethods
10
99
  private
@@ -13,10 +102,13 @@ module HTTPX
13
102
  return super(*args, **options) unless stream
14
103
 
15
104
  requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
16
-
17
105
  raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
18
106
 
19
- StreamResponse.new(requests.first, self)
107
+ request = requests.first
108
+
109
+ connections = _send_requests(requests, request.options)
110
+
111
+ StreamResponse.new(request, self, connections)
20
112
  end
21
113
  end
22
114
 
@@ -31,7 +123,7 @@ module HTTPX
31
123
  end
32
124
 
33
125
  module ResponseBodyMethods
34
- def initialize(*, **)
126
+ def initialize(*)
35
127
  super
36
128
  @stream = @response.stream
37
129
  end
@@ -51,78 +143,10 @@ module HTTPX
51
143
  end
52
144
  end
53
145
 
54
- class StreamResponse
55
- def initialize(request, session)
56
- @request = request
57
- @session = session
58
- @options = @request.options
59
- end
60
-
61
- def each(&block)
62
- return enum_for(__method__) unless block_given?
63
-
64
- raise Error, "response already streamed" if @response
65
-
66
- @request.stream = self
67
-
68
- begin
69
- @on_chunk = block
70
-
71
- response.raise_for_status
72
- response.close
73
- ensure
74
- @on_chunk = nil
75
- end
76
- end
77
-
78
- def each_line
79
- return enum_for(__method__) unless block_given?
80
-
81
- line = +""
82
-
83
- each do |chunk|
84
- line << chunk
85
-
86
- while (idx = line.index("\n"))
87
- yield line.byteslice(0..idx - 1)
88
-
89
- line = line.byteslice(idx + 1..-1)
90
- end
91
- end
92
- end
93
-
94
- # This is a ghost method. It's to be used ONLY internally, when processing streams
95
- def on_chunk(chunk)
96
- raise NoMethodError unless @on_chunk
97
-
98
- @on_chunk.call(chunk)
99
- end
100
-
101
- # :nocov:
102
- def inspect
103
- "#<StreamResponse:#{object_id}>"
104
- end
105
- # :nocov:
106
-
107
- def to_s
108
- response.to_s
109
- end
110
-
111
- private
112
-
113
- def response
114
- @response ||= @session.__send__(:send_requests, @request, @options).first
115
- end
116
-
117
- def respond_to_missing?(*args)
118
- @options.response_class.respond_to?(*args) || super
119
- end
120
-
121
- def method_missing(meth, *args, &block)
122
- return super unless @options.response_class.public_method_defined?(meth)
123
-
124
- response.__send__(meth, *args, &block)
125
- end
146
+ def self.const_missing(const_name)
147
+ super unless const_name == :StreamResponse
148
+ warn "DEPRECATION WARNING: the class #{self}::StreamResponse is deprecated. Use HTTPX::StreamResponse instead."
149
+ HTTPX::StreamResponse
126
150
  end
127
151
  end
128
152
  register_plugin :stream, Stream