httpx 0.8.0 → 0.10.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (138) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +9 -5
  4. data/doc/release_notes/0_0_1.md +7 -0
  5. data/doc/release_notes/0_0_2.md +9 -0
  6. data/doc/release_notes/0_0_3.md +9 -0
  7. data/doc/release_notes/0_0_4.md +7 -0
  8. data/doc/release_notes/0_0_5.md +5 -0
  9. data/doc/release_notes/0_10_0.md +66 -0
  10. data/doc/release_notes/0_10_1.md +39 -0
  11. data/doc/release_notes/0_1_0.md +9 -0
  12. data/doc/release_notes/0_2_0.md +5 -0
  13. data/doc/release_notes/0_2_1.md +16 -0
  14. data/doc/release_notes/0_3_0.md +12 -0
  15. data/doc/release_notes/0_3_1.md +6 -0
  16. data/doc/release_notes/0_4_0.md +51 -0
  17. data/doc/release_notes/0_4_1.md +3 -0
  18. data/doc/release_notes/0_5_0.md +15 -0
  19. data/doc/release_notes/0_5_1.md +14 -0
  20. data/doc/release_notes/0_6_0.md +5 -0
  21. data/doc/release_notes/0_6_1.md +6 -0
  22. data/doc/release_notes/0_6_2.md +6 -0
  23. data/doc/release_notes/0_6_3.md +13 -0
  24. data/doc/release_notes/0_6_4.md +21 -0
  25. data/doc/release_notes/0_6_5.md +22 -0
  26. data/doc/release_notes/0_6_6.md +19 -0
  27. data/doc/release_notes/0_6_7.md +5 -0
  28. data/doc/release_notes/0_7_0.md +46 -0
  29. data/doc/release_notes/0_8_0.md +27 -0
  30. data/doc/release_notes/0_8_1.md +8 -0
  31. data/doc/release_notes/0_8_2.md +7 -0
  32. data/doc/release_notes/0_9_0.md +38 -0
  33. data/lib/httpx.rb +2 -0
  34. data/lib/httpx/adapters/faraday.rb +1 -1
  35. data/lib/httpx/chainable.rb +11 -11
  36. data/lib/httpx/connection.rb +23 -31
  37. data/lib/httpx/connection/http1.rb +30 -4
  38. data/lib/httpx/connection/http2.rb +29 -10
  39. data/lib/httpx/domain_name.rb +440 -0
  40. data/lib/httpx/errors.rb +2 -1
  41. data/lib/httpx/extensions.rb +22 -2
  42. data/lib/httpx/headers.rb +2 -2
  43. data/lib/httpx/io/ssl.rb +0 -1
  44. data/lib/httpx/io/tcp.rb +6 -5
  45. data/lib/httpx/io/udp.rb +4 -1
  46. data/lib/httpx/options.rb +5 -1
  47. data/lib/httpx/parser/http1.rb +14 -17
  48. data/lib/httpx/plugins/compression.rb +46 -65
  49. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  50. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  51. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  52. data/lib/httpx/plugins/cookies.rb +21 -60
  53. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  54. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  55. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  56. data/lib/httpx/plugins/expect.rb +12 -1
  57. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  58. data/lib/httpx/plugins/h2c.rb +1 -1
  59. data/lib/httpx/plugins/multipart.rb +12 -6
  60. data/lib/httpx/plugins/persistent.rb +6 -1
  61. data/lib/httpx/plugins/proxy.rb +16 -2
  62. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  63. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  64. data/lib/httpx/plugins/retries.rb +3 -2
  65. data/lib/httpx/plugins/stream.rb +109 -13
  66. data/lib/httpx/pool.rb +14 -17
  67. data/lib/httpx/request.rb +8 -20
  68. data/lib/httpx/resolver.rb +7 -10
  69. data/lib/httpx/resolver/https.rb +22 -24
  70. data/lib/httpx/resolver/native.rb +19 -16
  71. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  72. data/lib/httpx/resolver/system.rb +2 -2
  73. data/lib/httpx/response.rb +16 -25
  74. data/lib/httpx/selector.rb +11 -18
  75. data/lib/httpx/session.rb +40 -26
  76. data/lib/httpx/transcoder.rb +18 -0
  77. data/lib/httpx/transcoder/chunker.rb +0 -2
  78. data/lib/httpx/transcoder/form.rb +9 -7
  79. data/lib/httpx/transcoder/json.rb +0 -4
  80. data/lib/httpx/utils.rb +45 -0
  81. data/lib/httpx/version.rb +1 -1
  82. data/sig/buffer.rbs +24 -0
  83. data/sig/callbacks.rbs +14 -0
  84. data/sig/chainable.rbs +37 -0
  85. data/sig/connection.rbs +85 -0
  86. data/sig/connection/http1.rbs +66 -0
  87. data/sig/connection/http2.rbs +78 -0
  88. data/sig/domain_name.rbs +17 -0
  89. data/sig/errors.rbs +3 -0
  90. data/sig/headers.rbs +42 -0
  91. data/sig/httpx.rbs +15 -0
  92. data/sig/loggable.rbs +11 -0
  93. data/sig/missing.rbs +12 -0
  94. data/sig/options.rbs +118 -0
  95. data/sig/parser/http1.rbs +50 -0
  96. data/sig/plugins/authentication.rbs +11 -0
  97. data/sig/plugins/basic_authentication.rbs +13 -0
  98. data/sig/plugins/compression.rbs +55 -0
  99. data/sig/plugins/compression/brotli.rbs +21 -0
  100. data/sig/plugins/compression/deflate.rbs +17 -0
  101. data/sig/plugins/compression/gzip.rbs +29 -0
  102. data/sig/plugins/cookies.rbs +26 -0
  103. data/sig/plugins/cookies/cookie.rbs +50 -0
  104. data/sig/plugins/cookies/jar.rbs +27 -0
  105. data/sig/plugins/digest_authentication.rbs +33 -0
  106. data/sig/plugins/expect.rbs +19 -0
  107. data/sig/plugins/follow_redirects.rbs +37 -0
  108. data/sig/plugins/h2c.rbs +26 -0
  109. data/sig/plugins/multipart.rbs +21 -0
  110. data/sig/plugins/persistent.rbs +17 -0
  111. data/sig/plugins/proxy.rbs +47 -0
  112. data/sig/plugins/proxy/http.rbs +14 -0
  113. data/sig/plugins/proxy/socks4.rbs +33 -0
  114. data/sig/plugins/proxy/socks5.rbs +36 -0
  115. data/sig/plugins/proxy/ssh.rbs +18 -0
  116. data/sig/plugins/push_promise.rbs +22 -0
  117. data/sig/plugins/rate_limiter.rbs +11 -0
  118. data/sig/plugins/retries.rbs +48 -0
  119. data/sig/plugins/stream.rbs +39 -0
  120. data/sig/pool.rbs +36 -0
  121. data/sig/registry.rbs +9 -0
  122. data/sig/request.rbs +61 -0
  123. data/sig/resolver.rbs +26 -0
  124. data/sig/resolver/https.rbs +49 -0
  125. data/sig/resolver/native.rbs +60 -0
  126. data/sig/resolver/resolver_mixin.rbs +27 -0
  127. data/sig/resolver/system.rbs +17 -0
  128. data/sig/response.rbs +87 -0
  129. data/sig/selector.rbs +20 -0
  130. data/sig/session.rbs +49 -0
  131. data/sig/timeout.rbs +29 -0
  132. data/sig/transcoder.rbs +18 -0
  133. data/sig/transcoder/body.rbs +18 -0
  134. data/sig/transcoder/chunker.rbs +32 -0
  135. data/sig/transcoder/form.rbs +16 -0
  136. data/sig/transcoder/json.rbs +14 -0
  137. metadata +128 -22
  138. data/lib/httpx/resolver/options.rb +0 -25
@@ -75,11 +75,11 @@ module HTTPX
75
75
  @status == 204 ||
76
76
  @status == 205 ||
77
77
  @status == 304 || begin
78
- content_length = @headers["content-length"]
79
- return false if content_length.nil?
78
+ content_length = @headers["content-length"]
79
+ return false if content_length.nil?
80
80
 
81
- content_length == "0"
82
- end
81
+ content_length == "0"
82
+ end
83
83
  end
84
84
 
85
85
  class Body
@@ -95,6 +95,8 @@ module HTTPX
95
95
  end
96
96
 
97
97
  def write(chunk)
98
+ return if @state == :closed
99
+
98
100
  @length += chunk.bytesize
99
101
  transition
100
102
  @buffer.write(chunk)
@@ -116,7 +118,7 @@ module HTTPX
116
118
  return enum_for(__method__) unless block_given?
117
119
 
118
120
  begin
119
- unless @state == :idle
121
+ if @buffer
120
122
  rewind
121
123
  while (chunk = @buffer.read(@window_size))
122
124
  yield(chunk.force_encoding(@encoding))
@@ -155,20 +157,19 @@ module HTTPX
155
157
  if dest.respond_to?(:path) && @buffer.respond_to?(:path)
156
158
  FileUtils.mv(@buffer.path, dest.path)
157
159
  else
158
- @buffer.rewind
159
160
  ::IO.copy_stream(@buffer, dest)
160
161
  end
161
162
  end
162
163
 
163
164
  # closes/cleans the buffer, resets everything
164
165
  def close
165
- return if @state == :idle
166
-
167
- @buffer.close
168
- @buffer.unlink if @buffer.respond_to?(:unlink)
169
- @buffer = nil
166
+ if @buffer
167
+ @buffer.close
168
+ @buffer.unlink if @buffer.respond_to?(:unlink)
169
+ @buffer = nil
170
+ end
170
171
  @length = 0
171
- @state = :idle
172
+ @state = :closed
172
173
  end
173
174
 
174
175
  def ==(other)
@@ -186,7 +187,7 @@ module HTTPX
186
187
  private
187
188
 
188
189
  def rewind
189
- return if @state == :idle
190
+ return unless @buffer
190
191
 
191
192
  @buffer.rewind
192
193
  end
@@ -239,13 +240,11 @@ module HTTPX
239
240
 
240
241
  private
241
242
 
242
- # :nodoc:
243
243
  def mime_type(str)
244
244
  m = str.to_s[MIME_TYPE_RE, 1]
245
245
  m && m.strip.downcase
246
246
  end
247
247
 
248
- # :nodoc:
249
248
  def charset(str)
250
249
  m = str.to_s[CHARSET_RE, 1]
251
250
  m && m.strip.delete('"')
@@ -269,15 +268,7 @@ module HTTPX
269
268
  @error.message
270
269
  end
271
270
 
272
- def reason
273
- @error.class.name
274
- end
275
-
276
- def headers
277
- {}
278
- end
279
-
280
- def body
271
+ def to_s
281
272
  @error.backtrace.join("\n")
282
273
  end
283
274
 
@@ -287,7 +278,7 @@ module HTTPX
287
278
 
288
279
  # rubocop:disable Style/MissingRespondToMissing
289
280
  def method_missing(meth, *, &block)
290
- raise NoMethodError, "undefined response method `#{meth}' for error response" if Response.public_method_defined?(meth)
281
+ raise NoMethodError, "undefined response method `#{meth}' for error response" if @options.response_class.public_method_defined?(meth)
291
282
 
292
283
  super
293
284
  end
@@ -2,21 +2,16 @@
2
2
 
3
3
  require "io/wait"
4
4
 
5
- module IOExtensions # :nodoc:
5
+ module IOExtensions
6
6
  refine IO do
7
- def wait(timeout = nil, mode = :read)
8
- case mode
9
- when :read
10
- wait_readable(timeout)
11
- when :write
12
- wait_writable(timeout)
13
- when :read_write
14
- r, w = IO.select([self], [self], nil, timeout)
15
-
16
- return unless r || w
17
-
18
- self
19
- end
7
+ # provides a fallback for rubies where IO#wait isn't implemented,
8
+ # but IO#wait_readable and IO#wait_writable are.
9
+ def wait(timeout = nil, _mode = :read_write)
10
+ r, w = IO.select([self], [self], nil, timeout)
11
+
12
+ return unless r || w
13
+
14
+ self
20
15
  end
21
16
  end
22
17
  end
@@ -51,7 +46,7 @@ class HTTPX::Selector
51
46
  READ_INTERESTS = %i[r rw].freeze
52
47
  WRITE_INTERESTS = %i[w rw].freeze
53
48
 
54
- def select_many(interval)
49
+ def select_many(interval, &block)
55
50
  selectables, r, w = nil
56
51
 
57
52
  # first, we group IOs based on interest type. On call to #interests however,
@@ -102,9 +97,7 @@ class HTTPX::Selector
102
97
  writers.delete(io)
103
98
  end if readers
104
99
 
105
- writers.each do |io|
106
- yield io
107
- end if writers
100
+ writers.each(&block) if writers
108
101
  end
109
102
 
110
103
  def select_one(interval)
@@ -5,7 +5,9 @@ module HTTPX
5
5
  include Loggable
6
6
  include Chainable
7
7
 
8
- def initialize(options = {}, &blk)
8
+ EMPTY_HASH = {}.freeze
9
+
10
+ def initialize(options = EMPTY_HASH, &blk)
9
11
  @options = self.class.default_options.merge(options)
10
12
  @responses = {}
11
13
  @persistent = @options.persistent
@@ -29,13 +31,21 @@ module HTTPX
29
31
  end
30
32
 
31
33
  def request(*args, **options)
32
- requests = build_requests(*args, options)
34
+ requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
33
35
  responses = send_requests(*requests, options)
34
36
  return responses.first if responses.size == 1
35
37
 
36
38
  responses
37
39
  end
38
40
 
41
+ def build_request(verb, uri, options = EMPTY_HASH)
42
+ rklass = @options.request_class
43
+ request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
44
+ request.on(:response, &method(:on_response).curry[request])
45
+ request.on(:promise, &method(:on_promise))
46
+ request
47
+ end
48
+
39
49
  private
40
50
 
41
51
  def pool
@@ -56,7 +66,8 @@ module HTTPX
56
66
  end
57
67
 
58
68
  def find_connection(request, connections, options)
59
- uri = URI(request.uri)
69
+ uri = request.uri
70
+
60
71
  connection = pool.find_connection(uri, options) || build_connection(uri, options)
61
72
  unless connections.nil? || connections.include?(connection)
62
73
  connections << connection
@@ -66,10 +77,16 @@ module HTTPX
66
77
  end
67
78
 
68
79
  def set_connection_callbacks(connection, connections, options)
69
- connection.on(:uncoalesce) do |uncoalesced_uri|
70
- other_connection = build_connection(uncoalesced_uri, options)
80
+ connection.on(:misdirected) do |misdirected_request|
81
+ other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
82
+ other_connection.merge(connection)
83
+ catch(:coalesced) do
84
+ pool.init_connection(other_connection, options)
85
+ end
86
+ set_connection_callbacks(other_connection, connections, options)
71
87
  connections << other_connection
72
- connection.unmerge(other_connection)
88
+ misdirected_request.transition(:idle)
89
+ other_connection.send(misdirected_request)
73
90
  end
74
91
  connection.on(:altsvc) do |alt_origin, origin, alt_params|
75
92
  other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
@@ -122,13 +139,13 @@ module HTTPX
122
139
  requests = case args.size
123
140
  when 1
124
141
  reqs = args.first
125
- reqs.map do |verb, uri|
126
- build_request(verb, uri, request_options)
142
+ reqs.map do |verb, uri, opts = EMPTY_HASH|
143
+ build_request(verb, uri, request_options.merge(opts))
127
144
  end
128
- when 2, 3
145
+ when 2
129
146
  verb, uris = args
130
147
  if uris.respond_to?(:each)
131
- uris.map do |uri, **opts|
148
+ uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
132
149
  build_request(verb, uri, request_options.merge(opts))
133
150
  end
134
151
  else
@@ -179,15 +196,13 @@ module HTTPX
179
196
  begin
180
197
  # guarantee ordered responses
181
198
  loop do
182
- begin
183
- request = requests.first
184
- pool.next_tick until (response = fetch_response(request, connections, request_options))
199
+ request = requests.first
200
+ pool.next_tick until (response = fetch_response(request, connections, request_options))
185
201
 
186
- responses << response
187
- requests.shift
202
+ responses << response
203
+ requests.shift
188
204
 
189
- break if requests.empty? || pool.empty?
190
- end
205
+ break if requests.empty? || pool.empty?
191
206
  end
192
207
  responses
193
208
  ensure
@@ -195,14 +210,6 @@ module HTTPX
195
210
  end
196
211
  end
197
212
 
198
- def build_request(verb, uri, options)
199
- rklass = @options.request_class
200
- request = rklass.new(verb, uri, @options.merge(options))
201
- request.on(:response, &method(:on_response).curry[request])
202
- request.on(:promise, &method(:on_promise))
203
- request
204
- end
205
-
206
213
  @default_options = Options.new
207
214
  @default_options.freeze
208
215
  @plugins = []
@@ -219,7 +226,7 @@ module HTTPX
219
226
  def plugin(pl, options = nil, &block)
220
227
  # raise Error, "Cannot add a plugin to a frozen config" if frozen?
221
228
  pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
222
- unless @plugins.include?(pl)
229
+ if !@plugins.include?(pl)
223
230
  @plugins << pl
224
231
  pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
225
232
  @default_options = @default_options.dup
@@ -243,6 +250,13 @@ module HTTPX
243
250
  opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
244
251
  pl.configure(self, &block) if pl.respond_to?(:configure)
245
252
 
253
+ @default_options.freeze
254
+ elsif options
255
+ # this can happen when two plugins are loaded, an one of them calls the other under the hood,
256
+ # albeit changing some default.
257
+ @default_options = @default_options.dup
258
+ @default_options = @default_options.merge(options)
259
+
246
260
  @default_options.freeze
247
261
  end
248
262
  self
@@ -3,6 +3,24 @@
3
3
  module HTTPX
4
4
  module Transcoder
5
5
  extend Registry
6
+
7
+ def self.normalize_keys(key, value, &block)
8
+ if value.respond_to?(:to_ary)
9
+ if value.empty?
10
+ block.call("#{key}[]")
11
+ else
12
+ value.to_ary.each do |element|
13
+ normalize_keys("#{key}[]", element, &block)
14
+ end
15
+ end
16
+ elsif value.respond_to?(:to_hash)
17
+ value.to_hash.each do |child_key, child_value|
18
+ normalize_keys("#{key}[#{child_key}]", child_value, &block)
19
+ end
20
+ else
21
+ block.call(key.to_s, value)
22
+ end
23
+ end
6
24
  end
7
25
  end
8
26
 
@@ -10,8 +10,6 @@ module HTTPX::Transcoder
10
10
  class Encoder
11
11
  extend Forwardable
12
12
 
13
- def_delegator :@raw, :readpartial
14
-
15
13
  def initialize(body)
16
14
  @raw = body
17
15
  end
@@ -12,21 +12,23 @@ module HTTPX::Transcoder
12
12
 
13
13
  def_delegator :@raw, :to_s
14
14
 
15
- def_delegator :@raw, :bytesize
15
+ def_delegator :@raw, :to_str
16
16
 
17
- def_delegator :@raw, :force_encoding
17
+ def_delegator :@raw, :bytesize
18
18
 
19
19
  def initialize(form)
20
- @raw = URI.encode_www_form(form)
20
+ @raw = form.each_with_object("".b) do |(key, val), buf|
21
+ HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
22
+ buf << "&" unless buf.empty?
23
+ buf << URI.encode_www_form_component(k)
24
+ buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
25
+ end
26
+ end
21
27
  end
22
28
 
23
29
  def content_type
24
30
  "application/x-www-form-urlencoded"
25
31
  end
26
-
27
- def to_str
28
- @raw.to_s
29
- end
30
32
  end
31
33
 
32
34
  def encode(form)
@@ -10,14 +10,10 @@ module HTTPX::Transcoder
10
10
  class Encoder
11
11
  extend Forwardable
12
12
 
13
- def_delegator :@raw, :to_str
14
-
15
13
  def_delegator :@raw, :to_s
16
14
 
17
15
  def_delegator :@raw, :bytesize
18
16
 
19
- def_delegator :@raw, :force_encoding
20
-
21
17
  def initialize(json)
22
18
  @raw = ::JSON.dump(json)
23
19
  @charset = @raw.encoding.name.downcase
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Utils
5
+ using URIExtensions
6
+
7
+ module_function
8
+
9
+ # The value of this field can be either an HTTP-date or a number of
10
+ # seconds to delay after the response is received.
11
+ def parse_retry_after(retry_after)
12
+ # first: bet on it being an integer
13
+ Integer(retry_after)
14
+ rescue ArgumentError
15
+ # Then it's a datetime
16
+ time = Time.httpdate(retry_after)
17
+ time - Time.now
18
+ end
19
+
20
+ if RUBY_VERSION < "2.3"
21
+ def uri(*args)
22
+ URI(*args)
23
+ end
24
+ else
25
+
26
+ URIParser = URI::RFC2396_Parser.new
27
+
28
+ def uri(uri)
29
+ return Kernel.URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
30
+
31
+ uri = Kernel.URI(URIParser.escape(uri))
32
+
33
+ non_ascii_hostname = URIParser.unescape(uri.host)
34
+
35
+ non_ascii_hostname.force_encoding(Encoding::UTF_8)
36
+
37
+ idna_hostname = DomainName.new(non_ascii_hostname).hostname
38
+
39
+ uri.host = idna_hostname
40
+ uri.non_ascii_hostname = non_ascii_hostname
41
+ uri
42
+ end
43
+ end
44
+ end
45
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- VERSION = "0.8.0"
4
+ VERSION = "0.10.1"
5
5
  end
@@ -0,0 +1,24 @@
1
+ module HTTPX
2
+ class Buffer
3
+ include _ToS
4
+ include _ToStr
5
+
6
+ @buffer: String
7
+
8
+ attr_reader limit: Integer
9
+
10
+ def full?: () -> bool
11
+ def shift!: (Integer) -> void
12
+
13
+ # delegated
14
+ def <<: (string data) -> void
15
+ def empty?: () -> bool
16
+ def bytesize: () -> Integer
17
+ def clear: () -> void
18
+ def replace: (string) -> void
19
+
20
+ private
21
+
22
+ def initialize: (Integer limit) -> untyped
23
+ end
24
+ end