httpx 0.9.0 → 0.11.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +13 -3
  4. data/doc/release_notes/0_10_0.md +66 -0
  5. data/doc/release_notes/0_10_1.md +37 -0
  6. data/doc/release_notes/0_10_2.md +5 -0
  7. data/doc/release_notes/0_11_0.md +76 -0
  8. data/doc/release_notes/0_11_1.md +1 -0
  9. data/lib/httpx.rb +2 -0
  10. data/lib/httpx/adapters/datadog.rb +205 -0
  11. data/lib/httpx/adapters/faraday.rb +1 -3
  12. data/lib/httpx/adapters/webmock.rb +123 -0
  13. data/lib/httpx/chainable.rb +10 -9
  14. data/lib/httpx/connection.rb +7 -24
  15. data/lib/httpx/connection/http1.rb +15 -2
  16. data/lib/httpx/connection/http2.rb +15 -16
  17. data/lib/httpx/domain_name.rb +438 -0
  18. data/lib/httpx/errors.rb +4 -1
  19. data/lib/httpx/extensions.rb +21 -1
  20. data/lib/httpx/headers.rb +1 -0
  21. data/lib/httpx/io/ssl.rb +4 -9
  22. data/lib/httpx/io/tcp.rb +6 -5
  23. data/lib/httpx/io/udp.rb +8 -4
  24. data/lib/httpx/options.rb +2 -0
  25. data/lib/httpx/parser/http1.rb +14 -17
  26. data/lib/httpx/plugins/compression.rb +28 -63
  27. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  28. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  29. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  30. data/lib/httpx/plugins/cookies.rb +21 -60
  31. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  32. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  33. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  34. data/lib/httpx/plugins/expect.rb +34 -11
  35. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  36. data/lib/httpx/plugins/h2c.rb +1 -1
  37. data/lib/httpx/plugins/multipart.rb +41 -30
  38. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  39. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  40. data/lib/httpx/plugins/multipart/part.rb +34 -0
  41. data/lib/httpx/plugins/persistent.rb +6 -1
  42. data/lib/httpx/plugins/proxy.rb +16 -2
  43. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  44. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  45. data/lib/httpx/plugins/push_promise.rb +2 -2
  46. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  47. data/lib/httpx/plugins/retries.rb +3 -2
  48. data/lib/httpx/plugins/stream.rb +109 -13
  49. data/lib/httpx/pool.rb +14 -20
  50. data/lib/httpx/request.rb +29 -31
  51. data/lib/httpx/resolver.rb +7 -6
  52. data/lib/httpx/resolver/https.rb +25 -25
  53. data/lib/httpx/resolver/native.rb +29 -22
  54. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  55. data/lib/httpx/resolver/system.rb +3 -3
  56. data/lib/httpx/response.rb +16 -23
  57. data/lib/httpx/selector.rb +11 -17
  58. data/lib/httpx/session.rb +39 -30
  59. data/lib/httpx/transcoder.rb +20 -0
  60. data/lib/httpx/transcoder/chunker.rb +0 -2
  61. data/lib/httpx/transcoder/form.rb +9 -7
  62. data/lib/httpx/transcoder/json.rb +0 -4
  63. data/lib/httpx/utils.rb +45 -0
  64. data/lib/httpx/version.rb +1 -1
  65. data/sig/buffer.rbs +24 -0
  66. data/sig/callbacks.rbs +14 -0
  67. data/sig/chainable.rbs +37 -0
  68. data/sig/connection.rbs +85 -0
  69. data/sig/connection/http1.rbs +66 -0
  70. data/sig/connection/http2.rbs +77 -0
  71. data/sig/domain_name.rbs +17 -0
  72. data/sig/errors.rbs +3 -0
  73. data/sig/headers.rbs +45 -0
  74. data/sig/httpx.rbs +15 -0
  75. data/sig/loggable.rbs +11 -0
  76. data/sig/options.rbs +118 -0
  77. data/sig/parser/http1.rbs +50 -0
  78. data/sig/plugins/authentication.rbs +11 -0
  79. data/sig/plugins/basic_authentication.rbs +13 -0
  80. data/sig/plugins/compression.rbs +55 -0
  81. data/sig/plugins/compression/brotli.rbs +21 -0
  82. data/sig/plugins/compression/deflate.rbs +17 -0
  83. data/sig/plugins/compression/gzip.rbs +29 -0
  84. data/sig/plugins/cookies.rbs +26 -0
  85. data/sig/plugins/cookies/cookie.rbs +50 -0
  86. data/sig/plugins/cookies/jar.rbs +27 -0
  87. data/sig/plugins/digest_authentication.rbs +33 -0
  88. data/sig/plugins/expect.rbs +19 -0
  89. data/sig/plugins/follow_redirects.rbs +37 -0
  90. data/sig/plugins/h2c.rbs +26 -0
  91. data/sig/plugins/multipart.rbs +44 -0
  92. data/sig/plugins/persistent.rbs +17 -0
  93. data/sig/plugins/proxy.rbs +47 -0
  94. data/sig/plugins/proxy/http.rbs +14 -0
  95. data/sig/plugins/proxy/socks4.rbs +33 -0
  96. data/sig/plugins/proxy/socks5.rbs +36 -0
  97. data/sig/plugins/proxy/ssh.rbs +18 -0
  98. data/sig/plugins/push_promise.rbs +22 -0
  99. data/sig/plugins/rate_limiter.rbs +11 -0
  100. data/sig/plugins/retries.rbs +48 -0
  101. data/sig/plugins/stream.rbs +39 -0
  102. data/sig/pool.rbs +36 -0
  103. data/sig/registry.rbs +9 -0
  104. data/sig/request.rbs +61 -0
  105. data/sig/resolver.rbs +26 -0
  106. data/sig/resolver/https.rbs +51 -0
  107. data/sig/resolver/native.rbs +60 -0
  108. data/sig/resolver/resolver_mixin.rbs +27 -0
  109. data/sig/resolver/system.rbs +17 -0
  110. data/sig/response.rbs +87 -0
  111. data/sig/selector.rbs +20 -0
  112. data/sig/session.rbs +49 -0
  113. data/sig/timeout.rbs +29 -0
  114. data/sig/transcoder.rbs +18 -0
  115. data/sig/transcoder/body.rbs +20 -0
  116. data/sig/transcoder/chunker.rbs +32 -0
  117. data/sig/transcoder/form.rbs +22 -0
  118. data/sig/transcoder/json.rbs +16 -0
  119. metadata +99 -59
  120. data/lib/httpx/resolver/options.rb +0 -25
@@ -38,7 +38,7 @@ module HTTPX
38
38
  def early_resolve(connection, hostname: connection.origin.host)
39
39
  addresses = connection.addresses ||
40
40
  ip_resolve(hostname) ||
41
- (@resolver_options.cache && Resolver.cached_lookup(hostname)) ||
41
+ (@resolver_options[:cache] && Resolver.cached_lookup(hostname)) ||
42
42
  system_resolve(hostname)
43
43
  return unless addresses
44
44
 
@@ -57,11 +57,13 @@ module HTTPX
57
57
  ips.map { |ip| IPAddr.new(ip) }
58
58
  end
59
59
 
60
- def emit_resolve_error(connection, hostname, ex = nil)
60
+ def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
61
61
  emit(:error, connection, resolve_error(hostname, ex))
62
62
  end
63
63
 
64
64
  def resolve_error(hostname, ex = nil)
65
+ return ex if ex.is_a?(ResolveError)
66
+
65
67
  message = ex ? ex.message : "Can't resolve #{hostname}"
66
68
  error = ResolveError.new(message)
67
69
  error.set_backtrace(ex ? ex.backtrace : caller)
@@ -14,13 +14,13 @@ module HTTPX
14
14
 
15
15
  def initialize(options)
16
16
  @options = Options.new(options)
17
- @resolver_options = Resolver::Options.new(@options.resolver_options)
17
+ @resolver_options = @options.resolver_options
18
18
  @state = :idle
19
- resolv_options = @resolver_options.to_h
19
+ resolv_options = @resolver_options.dup
20
20
  timeouts = resolv_options.delete(:timeouts)
21
21
  resolv_options.delete(:cache)
22
22
  @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
23
- @resolver.timeouts = timeouts if timeouts
23
+ @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
24
24
  end
25
25
 
26
26
  def closed?
@@ -75,11 +75,11 @@ module HTTPX
75
75
  @status == 204 ||
76
76
  @status == 205 ||
77
77
  @status == 304 || begin
78
- content_length = @headers["content-length"]
79
- return false if content_length.nil?
78
+ content_length = @headers["content-length"]
79
+ return false if content_length.nil?
80
80
 
81
- content_length == "0"
82
- end
81
+ content_length == "0"
82
+ end
83
83
  end
84
84
 
85
85
  class Body
@@ -95,6 +95,8 @@ module HTTPX
95
95
  end
96
96
 
97
97
  def write(chunk)
98
+ return if @state == :closed
99
+
98
100
  @length += chunk.bytesize
99
101
  transition
100
102
  @buffer.write(chunk)
@@ -116,7 +118,7 @@ module HTTPX
116
118
  return enum_for(__method__) unless block_given?
117
119
 
118
120
  begin
119
- unless @state == :idle
121
+ if @buffer
120
122
  rewind
121
123
  while (chunk = @buffer.read(@window_size))
122
124
  yield(chunk.force_encoding(@encoding))
@@ -155,20 +157,19 @@ module HTTPX
155
157
  if dest.respond_to?(:path) && @buffer.respond_to?(:path)
156
158
  FileUtils.mv(@buffer.path, dest.path)
157
159
  else
158
- @buffer.rewind
159
160
  ::IO.copy_stream(@buffer, dest)
160
161
  end
161
162
  end
162
163
 
163
164
  # closes/cleans the buffer, resets everything
164
165
  def close
165
- return if @state == :idle
166
-
167
- @buffer.close
168
- @buffer.unlink if @buffer.respond_to?(:unlink)
169
- @buffer = nil
166
+ if @buffer
167
+ @buffer.close
168
+ @buffer.unlink if @buffer.respond_to?(:unlink)
169
+ @buffer = nil
170
+ end
170
171
  @length = 0
171
- @state = :idle
172
+ @state = :closed
172
173
  end
173
174
 
174
175
  def ==(other)
@@ -186,7 +187,7 @@ module HTTPX
186
187
  private
187
188
 
188
189
  def rewind
189
- return if @state == :idle
190
+ return unless @buffer
190
191
 
191
192
  @buffer.rewind
192
193
  end
@@ -267,15 +268,7 @@ module HTTPX
267
268
  @error.message
268
269
  end
269
270
 
270
- def reason
271
- @error.class.name
272
- end
273
-
274
- def headers
275
- {}
276
- end
277
-
278
- def body
271
+ def to_s
279
272
  @error.backtrace.join("\n")
280
273
  end
281
274
 
@@ -285,7 +278,7 @@ module HTTPX
285
278
 
286
279
  # rubocop:disable Style/MissingRespondToMissing
287
280
  def method_missing(meth, *, &block)
288
- raise NoMethodError, "undefined response method `#{meth}' for error response" if Response.public_method_defined?(meth)
281
+ raise NoMethodError, "undefined response method `#{meth}' for error response" if @options.response_class.public_method_defined?(meth)
289
282
 
290
283
  super
291
284
  end
@@ -4,19 +4,14 @@ require "io/wait"
4
4
 
5
5
  module IOExtensions
6
6
  refine IO do
7
- def wait(timeout = nil, mode = :read)
8
- case mode
9
- when :read
10
- wait_readable(timeout)
11
- when :write
12
- wait_writable(timeout)
13
- when :read_write
14
- r, w = IO.select([self], [self], nil, timeout)
15
-
16
- return unless r || w
17
-
18
- self
19
- end
7
+ # provides a fallback for rubies where IO#wait isn't implemented,
8
+ # but IO#wait_readable and IO#wait_writable are.
9
+ def wait(timeout = nil, _mode = :read_write)
10
+ r, w = IO.select([self], [self], nil, timeout)
11
+
12
+ return unless r || w
13
+
14
+ self
20
15
  end
21
16
  end
22
17
  end
@@ -51,7 +46,7 @@ class HTTPX::Selector
51
46
  READ_INTERESTS = %i[r rw].freeze
52
47
  WRITE_INTERESTS = %i[w rw].freeze
53
48
 
54
- def select_many(interval)
49
+ def select_many(interval, &block)
55
50
  selectables, r, w = nil
56
51
 
57
52
  # first, we group IOs based on interest type. On call to #interests however,
@@ -102,9 +97,7 @@ class HTTPX::Selector
102
97
  writers.delete(io)
103
98
  end if readers
104
99
 
105
- writers.each do |io|
106
- yield io
107
- end if writers
100
+ writers.each(&block) if writers
108
101
  end
109
102
 
110
103
  def select_one(interval)
@@ -124,6 +117,7 @@ class HTTPX::Selector
124
117
  yield io
125
118
  rescue IOError, SystemCallError
126
119
  @selectables.reject!(&:closed?)
120
+ raise unless @selectables.empty?
127
121
  end
128
122
 
129
123
  def select(interval, &block)
@@ -41,7 +41,7 @@ module HTTPX
41
41
  def build_request(verb, uri, options = EMPTY_HASH)
42
42
  rklass = @options.request_class
43
43
  request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
44
- request.on(:response, &method(:on_response).curry[request])
44
+ request.on(:response, &method(:on_response).curry(2)[request])
45
45
  request.on(:promise, &method(:on_promise))
46
46
  request
47
47
  end
@@ -66,7 +66,8 @@ module HTTPX
66
66
  end
67
67
 
68
68
  def find_connection(request, connections, options)
69
- uri = URI(request.uri)
69
+ uri = request.uri
70
+
70
71
  connection = pool.find_connection(uri, options) || build_connection(uri, options)
71
72
  unless connections.nil? || connections.include?(connection)
72
73
  connections << connection
@@ -76,10 +77,16 @@ module HTTPX
76
77
  end
77
78
 
78
79
  def set_connection_callbacks(connection, connections, options)
79
- connection.on(:uncoalesce) do |uncoalesced_uri|
80
- other_connection = build_connection(uncoalesced_uri, options)
80
+ connection.on(:misdirected) do |misdirected_request|
81
+ other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
82
+ other_connection.merge(connection)
83
+ catch(:coalesced) do
84
+ pool.init_connection(other_connection, options)
85
+ end
86
+ set_connection_callbacks(other_connection, connections, options)
81
87
  connections << other_connection
82
- connection.unmerge(other_connection)
88
+ misdirected_request.transition(:idle)
89
+ other_connection.send(misdirected_request)
83
90
  end
84
91
  connection.on(:altsvc) do |alt_origin, origin, alt_params|
85
92
  other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
@@ -129,23 +136,20 @@ module HTTPX
129
136
  def build_requests(*args, options)
130
137
  request_options = @options.merge(options)
131
138
 
132
- requests = case args.size
133
- when 1
134
- reqs = args.first
135
- reqs.map do |verb, uri, opts = EMPTY_HASH|
136
- build_request(verb, uri, request_options.merge(opts))
137
- end
138
- when 2, 3
139
- verb, uris = args
140
- if uris.respond_to?(:each)
141
- uris.map do |uri, **opts|
142
- build_request(verb, uri, request_options.merge(opts))
143
- end
144
- else
145
- [build_request(verb, uris, request_options)]
146
- end
147
- else
148
- raise ArgumentError, "unsupported number of arguments"
139
+ requests = if args.size == 1
140
+ reqs = args.first
141
+ reqs.map do |verb, uri, opts = EMPTY_HASH|
142
+ build_request(verb, uri, request_options.merge(opts))
143
+ end
144
+ else
145
+ verb, uris = args
146
+ if uris.respond_to?(:each)
147
+ uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
148
+ build_request(verb, uri, request_options.merge(opts))
149
+ end
150
+ else
151
+ [build_request(verb, uris, request_options)]
152
+ end
149
153
  end
150
154
  raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
151
155
 
@@ -189,15 +193,13 @@ module HTTPX
189
193
  begin
190
194
  # guarantee ordered responses
191
195
  loop do
192
- begin
193
- request = requests.first
194
- pool.next_tick until (response = fetch_response(request, connections, request_options))
196
+ request = requests.first
197
+ pool.next_tick until (response = fetch_response(request, connections, request_options))
195
198
 
196
- responses << response
197
- requests.shift
199
+ responses << response
200
+ requests.shift
198
201
 
199
- break if requests.empty? || pool.empty?
200
- end
202
+ break if requests.empty? || pool.empty?
201
203
  end
202
204
  responses
203
205
  ensure
@@ -221,7 +223,7 @@ module HTTPX
221
223
  def plugin(pl, options = nil, &block)
222
224
  # raise Error, "Cannot add a plugin to a frozen config" if frozen?
223
225
  pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
224
- unless @plugins.include?(pl)
226
+ if !@plugins.include?(pl)
225
227
  @plugins << pl
226
228
  pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
227
229
  @default_options = @default_options.dup
@@ -245,6 +247,13 @@ module HTTPX
245
247
  opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
246
248
  pl.configure(self, &block) if pl.respond_to?(:configure)
247
249
 
250
+ @default_options.freeze
251
+ elsif options
252
+ # this can happen when two plugins are loaded, an one of them calls the other under the hood,
253
+ # albeit changing some default.
254
+ @default_options = @default_options.dup
255
+ @default_options = @default_options.merge(options)
256
+
248
257
  @default_options.freeze
249
258
  end
250
259
  self
@@ -3,6 +3,26 @@
3
3
  module HTTPX
4
4
  module Transcoder
5
5
  extend Registry
6
+
7
+ def self.normalize_keys(key, value, cond = nil, &block)
8
+ if (cond && cond.call(value))
9
+ block.call(key.to_s, value)
10
+ elsif value.respond_to?(:to_ary)
11
+ if value.empty?
12
+ block.call("#{key}[]")
13
+ else
14
+ value.to_ary.each do |element|
15
+ normalize_keys("#{key}[]", element, cond, &block)
16
+ end
17
+ end
18
+ elsif value.respond_to?(:to_hash)
19
+ value.to_hash.each do |child_key, child_value|
20
+ normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
21
+ end
22
+ else
23
+ block.call(key.to_s, value)
24
+ end
25
+ end
6
26
  end
7
27
  end
8
28
 
@@ -10,8 +10,6 @@ module HTTPX::Transcoder
10
10
  class Encoder
11
11
  extend Forwardable
12
12
 
13
- def_delegator :@raw, :readpartial
14
-
15
13
  def initialize(body)
16
14
  @raw = body
17
15
  end
@@ -12,21 +12,23 @@ module HTTPX::Transcoder
12
12
 
13
13
  def_delegator :@raw, :to_s
14
14
 
15
- def_delegator :@raw, :bytesize
15
+ def_delegator :@raw, :to_str
16
16
 
17
- def_delegator :@raw, :force_encoding
17
+ def_delegator :@raw, :bytesize
18
18
 
19
19
  def initialize(form)
20
- @raw = URI.encode_www_form(form)
20
+ @raw = form.each_with_object("".b) do |(key, val), buf|
21
+ HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
22
+ buf << "&" unless buf.empty?
23
+ buf << URI.encode_www_form_component(k)
24
+ buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
25
+ end
26
+ end
21
27
  end
22
28
 
23
29
  def content_type
24
30
  "application/x-www-form-urlencoded"
25
31
  end
26
-
27
- def to_str
28
- @raw.to_s
29
- end
30
32
  end
31
33
 
32
34
  def encode(form)
@@ -10,14 +10,10 @@ module HTTPX::Transcoder
10
10
  class Encoder
11
11
  extend Forwardable
12
12
 
13
- def_delegator :@raw, :to_str
14
-
15
13
  def_delegator :@raw, :to_s
16
14
 
17
15
  def_delegator :@raw, :bytesize
18
16
 
19
- def_delegator :@raw, :force_encoding
20
-
21
17
  def initialize(json)
22
18
  @raw = ::JSON.dump(json)
23
19
  @charset = @raw.encoding.name.downcase
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Utils
5
+ using URIExtensions
6
+
7
+ module_function
8
+
9
+ # The value of this field can be either an HTTP-date or a number of
10
+ # seconds to delay after the response is received.
11
+ def parse_retry_after(retry_after)
12
+ # first: bet on it being an integer
13
+ Integer(retry_after)
14
+ rescue ArgumentError
15
+ # Then it's a datetime
16
+ time = Time.httpdate(retry_after)
17
+ time - Time.now
18
+ end
19
+
20
+ if RUBY_VERSION < "2.3"
21
+ def uri(*args)
22
+ URI(*args)
23
+ end
24
+ else
25
+
26
+ URIParser = URI::RFC2396_Parser.new
27
+
28
+ def uri(uri)
29
+ return Kernel.URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
30
+
31
+ uri = Kernel.URI(URIParser.escape(uri))
32
+
33
+ non_ascii_hostname = URIParser.unescape(uri.host)
34
+
35
+ non_ascii_hostname.force_encoding(Encoding::UTF_8)
36
+
37
+ idna_hostname = DomainName.new(non_ascii_hostname).hostname
38
+
39
+ uri.host = idna_hostname
40
+ uri.non_ascii_hostname = non_ascii_hostname
41
+ uri
42
+ end
43
+ end
44
+ end
45
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- VERSION = "0.9.0"
4
+ VERSION = "0.11.1"
5
5
  end
@@ -0,0 +1,24 @@
1
+ module HTTPX
2
+ class Buffer
3
+ include _ToS
4
+ include _ToStr
5
+
6
+ @buffer: String
7
+
8
+ attr_reader limit: Integer
9
+
10
+ def full?: () -> bool
11
+ def shift!: (Integer) -> void
12
+
13
+ # delegated
14
+ def <<: (string data) -> void
15
+ def empty?: () -> bool
16
+ def bytesize: () -> Integer
17
+ def clear: () -> void
18
+ def replace: (string) -> void
19
+
20
+ private
21
+
22
+ def initialize: (Integer limit) -> untyped
23
+ end
24
+ end