httpx 0.6.7 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +7 -5
  3. data/doc/release_notes/0_0_1.md +7 -0
  4. data/doc/release_notes/0_0_2.md +9 -0
  5. data/doc/release_notes/0_0_3.md +9 -0
  6. data/doc/release_notes/0_0_4.md +7 -0
  7. data/doc/release_notes/0_0_5.md +5 -0
  8. data/doc/release_notes/0_1_0.md +9 -0
  9. data/doc/release_notes/0_2_0.md +5 -0
  10. data/doc/release_notes/0_2_1.md +16 -0
  11. data/doc/release_notes/0_3_0.md +12 -0
  12. data/doc/release_notes/0_3_1.md +6 -0
  13. data/doc/release_notes/0_4_0.md +51 -0
  14. data/doc/release_notes/0_4_1.md +3 -0
  15. data/doc/release_notes/0_5_0.md +15 -0
  16. data/doc/release_notes/0_5_1.md +14 -0
  17. data/doc/release_notes/0_6_0.md +5 -0
  18. data/doc/release_notes/0_6_1.md +6 -0
  19. data/doc/release_notes/0_6_2.md +6 -0
  20. data/doc/release_notes/0_6_3.md +13 -0
  21. data/doc/release_notes/0_6_4.md +21 -0
  22. data/doc/release_notes/0_6_5.md +22 -0
  23. data/doc/release_notes/0_6_6.md +19 -0
  24. data/doc/release_notes/0_6_7.md +5 -0
  25. data/doc/release_notes/0_7_0.md +46 -0
  26. data/doc/release_notes/0_8_0.md +27 -0
  27. data/doc/release_notes/0_8_1.md +8 -0
  28. data/doc/release_notes/0_8_2.md +7 -0
  29. data/doc/release_notes/0_9_0.md +38 -0
  30. data/lib/httpx/adapters/faraday.rb +2 -2
  31. data/lib/httpx/altsvc.rb +18 -2
  32. data/lib/httpx/chainable.rb +27 -9
  33. data/lib/httpx/connection.rb +215 -65
  34. data/lib/httpx/connection/http1.rb +54 -18
  35. data/lib/httpx/connection/http2.rb +100 -37
  36. data/lib/httpx/extensions.rb +2 -2
  37. data/lib/httpx/headers.rb +2 -2
  38. data/lib/httpx/io/ssl.rb +11 -3
  39. data/lib/httpx/io/tcp.rb +12 -2
  40. data/lib/httpx/loggable.rb +6 -6
  41. data/lib/httpx/options.rb +43 -28
  42. data/lib/httpx/plugins/authentication.rb +1 -1
  43. data/lib/httpx/plugins/compression.rb +28 -8
  44. data/lib/httpx/plugins/compression/gzip.rb +22 -12
  45. data/lib/httpx/plugins/cookies.rb +12 -8
  46. data/lib/httpx/plugins/digest_authentication.rb +2 -0
  47. data/lib/httpx/plugins/expect.rb +79 -0
  48. data/lib/httpx/plugins/follow_redirects.rb +1 -2
  49. data/lib/httpx/plugins/h2c.rb +0 -1
  50. data/lib/httpx/plugins/proxy.rb +23 -20
  51. data/lib/httpx/plugins/proxy/http.rb +9 -6
  52. data/lib/httpx/plugins/proxy/socks4.rb +1 -1
  53. data/lib/httpx/plugins/proxy/socks5.rb +5 -1
  54. data/lib/httpx/plugins/proxy/ssh.rb +0 -4
  55. data/lib/httpx/plugins/push_promise.rb +2 -2
  56. data/lib/httpx/plugins/retries.rb +32 -29
  57. data/lib/httpx/pool.rb +15 -10
  58. data/lib/httpx/registry.rb +2 -1
  59. data/lib/httpx/request.rb +8 -6
  60. data/lib/httpx/resolver.rb +7 -8
  61. data/lib/httpx/resolver/https.rb +15 -3
  62. data/lib/httpx/resolver/native.rb +22 -32
  63. data/lib/httpx/resolver/options.rb +2 -2
  64. data/lib/httpx/resolver/resolver_mixin.rb +1 -1
  65. data/lib/httpx/response.rb +17 -3
  66. data/lib/httpx/selector.rb +96 -95
  67. data/lib/httpx/session.rb +33 -34
  68. data/lib/httpx/timeout.rb +7 -1
  69. data/lib/httpx/version.rb +1 -1
  70. metadata +77 -20
@@ -2,7 +2,7 @@
2
2
 
3
3
  module HTTPX
4
4
  class Headers
5
- EMPTY = [].freeze # :nodoc:
5
+ EMPTY = [].freeze
6
6
 
7
7
  class << self
8
8
  def new(headers = nil)
@@ -67,7 +67,7 @@ module HTTPX
67
67
  #
68
68
  def [](field)
69
69
  a = @headers[downcased(field)] || return
70
- a.join(",")
70
+ a.join(", ")
71
71
  end
72
72
 
73
73
  # sets +value+ (if not nil) as single value for the +field+ header.
@@ -20,6 +20,10 @@ module HTTPX
20
20
  @state = :negotiated if @keep_open
21
21
  end
22
22
 
23
+ def interests
24
+ @interests || super
25
+ end
26
+
23
27
  def protocol
24
28
  @io.alpn_protocol || super
25
29
  rescue StandardError
@@ -62,15 +66,18 @@ module HTTPX
62
66
  @io.connect_nonblock
63
67
  @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
64
68
  transition(:negotiated)
65
- rescue ::IO::WaitReadable,
66
- ::IO::WaitWritable
69
+ rescue ::IO::WaitReadable
70
+ @interests = :r
71
+ rescue ::IO::WaitWritable
72
+ @interests = :w
67
73
  end
68
74
 
69
75
  # :nocov:
70
76
  if RUBY_VERSION < "2.3"
71
- def read(*)
77
+ def read(_, buffer)
72
78
  super
73
79
  rescue ::IO::WaitWritable
80
+ buffer.clear
74
81
  0
75
82
  end
76
83
 
@@ -86,6 +93,7 @@ module HTTPX
86
93
  buffer.bytesize
87
94
  rescue ::IO::WaitReadable,
88
95
  ::IO::WaitWritable
96
+ buffer.clear
89
97
  0
90
98
  rescue EOFError
91
99
  nil
@@ -11,6 +11,8 @@ module HTTPX
11
11
 
12
12
  attr_reader :addresses
13
13
 
14
+ attr_reader :state
15
+
14
16
  alias_method :host, :ip
15
17
 
16
18
  def initialize(origin, addresses, options)
@@ -41,6 +43,10 @@ module HTTPX
41
43
  @io ||= build_socket
42
44
  end
43
45
 
46
+ def interests
47
+ :w
48
+ end
49
+
44
50
  def to_io
45
51
  @io.to_io
46
52
  end
@@ -67,7 +73,7 @@ module HTTPX
67
73
  @ip_index -= 1
68
74
  retry
69
75
  rescue Errno::ETIMEDOUT => e
70
- raise ConnectTimeout, e.message if @ip_index <= 0
76
+ raise ConnectTimeoutError, e.message if @ip_index <= 0
71
77
 
72
78
  @ip_index -= 1
73
79
  retry
@@ -82,6 +88,7 @@ module HTTPX
82
88
  @io.read_nonblock(size, buffer)
83
89
  buffer.bytesize
84
90
  rescue ::IO::WaitReadable
91
+ buffer.clear
85
92
  0
86
93
  rescue EOFError
87
94
  nil
@@ -100,7 +107,10 @@ module HTTPX
100
107
  else
101
108
  def read(size, buffer)
102
109
  ret = @io.read_nonblock(size, buffer, exception: false)
103
- return 0 if ret == :wait_readable
110
+ if ret == :wait_readable
111
+ buffer.clear
112
+ return 0
113
+ end
104
114
  return if ret.nil?
105
115
 
106
116
  buffer.bytesize
@@ -13,35 +13,35 @@ module HTTPX
13
13
  white: 37,
14
14
  }.freeze
15
15
 
16
- def log(level: @options.debug_level, label: "", color: nil, &msg)
16
+ def log(level: @options.debug_level, color: nil, &msg)
17
17
  return unless @options.debug
18
18
  return unless @options.debug_level >= level
19
19
 
20
20
  debug_stream = @options.debug
21
21
 
22
- message = (+label << msg.call << "\n")
22
+ message = (+"" << msg.call << "\n")
23
23
  message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
24
24
  debug_stream << message
25
25
  end
26
26
 
27
27
  if !Exception.instance_methods.include?(:full_message)
28
28
 
29
- def log_exception(ex, level: @options.debug_level, label: "", color: nil)
29
+ def log_exception(ex, level: @options.debug_level, color: nil)
30
30
  return unless @options.debug
31
31
  return unless @options.debug_level >= level
32
32
 
33
33
  message = +"#{ex.message} (#{ex.class})"
34
34
  message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
35
- log(level: level, label: label, color: color) { message }
35
+ log(level: level, color: color) { message }
36
36
  end
37
37
 
38
38
  else
39
39
 
40
- def log_exception(ex, level: @options.debug_level, label: "", color: nil)
40
+ def log_exception(ex, level: @options.debug_level, color: nil)
41
41
  return unless @options.debug
42
42
  return unless @options.debug_level >= level
43
43
 
44
- log(level: level, label: label, color: color) { ex.full_message }
44
+ log(level: level, color: color) { ex.full_message }
45
45
  end
46
46
 
47
47
  end
@@ -2,7 +2,6 @@
2
2
 
3
3
  module HTTPX
4
4
  class Options
5
- MAX_CONCURRENT_REQUESTS = 100
6
5
  WINDOW_SIZE = 1 << 14 # 16K
7
6
  MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
8
7
 
@@ -26,16 +25,28 @@ module HTTPX
26
25
 
27
26
  def def_option(name, &interpreter)
28
27
  defined_options << name.to_sym
29
- interpreter ||= ->(v) { v }
30
28
 
31
- attr_accessor name
32
- protected :"#{name}="
29
+ attr_reader name
30
+
31
+ if interpreter
32
+ define_method(:"#{name}=") do |value|
33
+ return if value.nil?
34
+
35
+ instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
36
+ end
37
+
38
+ define_method(:"with_#{name}") do |value|
39
+ merge(name => instance_exec(value, &interpreter))
40
+ end
41
+ else
42
+ attr_writer name
33
43
 
34
- define_method(:"with_#{name}") do |value|
35
- other = dup
36
- other.send(:"#{name}=", other.instance_exec(value, &interpreter))
37
- other
44
+ define_method(:"with_#{name}") do |value|
45
+ merge(name => value)
46
+ end
38
47
  end
48
+
49
+ protected :"#{name}="
39
50
  end
40
51
  end
41
52
 
@@ -48,7 +59,6 @@ module HTTPX
48
59
  :fallback_protocol => "http/1.1",
49
60
  :timeout => Timeout.new,
50
61
  :headers => {},
51
- :max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
52
62
  :window_size => WINDOW_SIZE,
53
63
  :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
54
64
  :request_class => Class.new(Request),
@@ -65,43 +75,52 @@ module HTTPX
65
75
  }
66
76
 
67
77
  defaults.merge!(options)
68
- defaults[:headers] = Headers.new(defaults[:headers])
69
- defaults.each { |(k, v)| self[k] = v }
78
+ defaults.each do |(k, v)|
79
+ __send__(:"#{k}=", v)
80
+ end
70
81
  end
71
82
 
72
83
  def_option(:headers) do |headers|
73
- self.headers.merge(headers)
84
+ if self.headers
85
+ self.headers.merge(headers)
86
+ else
87
+ Headers.new(headers)
88
+ end
74
89
  end
75
90
 
76
91
  def_option(:timeout) do |opts|
77
- self.timeout = Timeout.new(opts)
92
+ Timeout.new(opts)
78
93
  end
79
94
 
80
95
  def_option(:max_concurrent_requests) do |num|
81
- max = Integer(num)
82
- raise Error, ":max_concurrent_requests must be positive" unless max.positive?
96
+ raise Error, ":max_concurrent_requests must be positive" unless num.positive?
83
97
 
84
- self.max_concurrent_requests = max
98
+ num
99
+ end
100
+
101
+ def_option(:max_requests) do |num|
102
+ raise Error, ":max_requests must be positive" unless num.positive?
103
+
104
+ num
85
105
  end
86
106
 
87
107
  def_option(:window_size) do |num|
88
- self.window_size = Integer(num)
108
+ Integer(num)
89
109
  end
90
110
 
91
111
  def_option(:body_threshold_size) do |num|
92
- self.body_threshold_size = Integer(num)
112
+ Integer(num)
93
113
  end
94
114
 
95
115
  def_option(:transport) do |tr|
96
116
  transport = tr.to_s
97
117
  raise Error, "#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
98
118
 
99
- self.transport = transport
119
+ transport
100
120
  end
101
121
 
102
122
  %w[
103
- params form json body
104
- follow ssl http2_settings
123
+ params form json body ssl http2_settings
105
124
  request_class response_class headers_class request_body_class response_body_class connection_class
106
125
  io fallback_protocol debug debug_level transport_options resolver_class resolver_options
107
126
  persistent
@@ -127,8 +146,10 @@ module HTTPX
127
146
  end
128
147
 
129
148
  def merge(other)
130
- h1 = to_hash
131
149
  h2 = other.to_hash
150
+ return self if h2.empty?
151
+
152
+ h1 = to_hash
132
153
 
133
154
  merged = h1.merge(h2) do |k, v1, v2|
134
155
  case k
@@ -172,11 +193,5 @@ module HTTPX
172
193
  response_body_class.freeze
173
194
  connection_class.freeze
174
195
  end
175
-
176
- protected
177
-
178
- def []=(option, val)
179
- send(:"#{option}=", val)
180
- end
181
196
  end
182
197
  end
@@ -11,7 +11,7 @@ module HTTPX
11
11
  module Authentication
12
12
  module InstanceMethods
13
13
  def authentication(token)
14
- headers("authorization" => token)
14
+ with(headers: { "authorization" => token })
15
15
  end
16
16
  end
17
17
  end
@@ -14,13 +14,23 @@ module HTTPX
14
14
  #
15
15
  module Compression
16
16
  extend Registry
17
- def self.load_dependencies(klass)
18
- klass.plugin(:"compression/gzip")
19
- klass.plugin(:"compression/deflate")
20
- end
21
17
 
22
- def self.extra_options(options)
23
- options.merge(headers: { "accept-encoding" => Compression.registry.keys })
18
+ class << self
19
+ def load_dependencies(klass)
20
+ klass.plugin(:"compression/gzip")
21
+ klass.plugin(:"compression/deflate")
22
+ end
23
+
24
+ def extra_options(options)
25
+ Class.new(options.class) do
26
+ def_option(:compression_threshold_size) do |bytes|
27
+ bytes = Integer(bytes)
28
+ raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
29
+
30
+ bytes
31
+ end
32
+ end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
33
+ end
24
34
  end
25
35
 
26
36
  module RequestMethods
@@ -32,11 +42,19 @@ module HTTPX
32
42
  end
33
43
 
34
44
  module RequestBodyMethods
35
- def initialize(*)
45
+ def initialize(*, options)
36
46
  super
37
47
  return if @body.nil?
38
48
 
49
+ if (threshold = options.compression_threshold_size)
50
+ unless unbounded_body?
51
+ return if @body.bytesize < threshold
52
+ end
53
+ end
54
+
39
55
  @headers.get("content-encoding").each do |encoding|
56
+ next if encoding == "identity"
57
+
40
58
  @body = Encoder.new(@body, Compression.registry(encoding).encoder)
41
59
  end
42
60
  @headers["content-length"] = @body.bytesize unless chunked?
@@ -54,6 +72,8 @@ module HTTPX
54
72
  return unless @headers.key?("content-encoding")
55
73
 
56
74
  @_decoders = @headers.get("content-encoding").map do |encoding|
75
+ next if encoding == "identity"
76
+
57
77
  decoder = Compression.registry(encoding).decoder
58
78
  # do not uncompress if there is no decoder available. In fact, we can't reliably
59
79
  # continue decompressing beyond that, so ignore.
@@ -61,7 +81,7 @@ module HTTPX
61
81
 
62
82
  @encodings << encoding
63
83
  decoder
64
- end
84
+ end.compact
65
85
 
66
86
  # remove encodings that we are able to decode
67
87
  @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
@@ -15,31 +15,41 @@ module HTTPX
15
15
  end
16
16
 
17
17
  class Encoder
18
+ def initialize
19
+ @compressed_chunk = "".b
20
+ end
21
+
18
22
  def deflate(raw, buffer, chunk_size:)
19
23
  gzip = Zlib::GzipWriter.new(self)
20
24
 
21
- while (chunk = raw.read(chunk_size))
22
- gzip.write(chunk)
23
- gzip.flush
24
- compressed = compressed_chunk
25
- buffer << compressed
26
- yield compressed if block_given?
25
+ begin
26
+ while (chunk = raw.read(chunk_size))
27
+ gzip.write(chunk)
28
+ gzip.flush
29
+ compressed = compressed_chunk
30
+ buffer << compressed
31
+ yield compressed if block_given?
32
+ end
33
+ ensure
34
+ gzip.close
27
35
  end
28
- ensure
29
- gzip.close
36
+
37
+ return unless (compressed = compressed_chunk)
38
+
39
+ buffer << compressed
40
+ yield compressed if block_given?
30
41
  end
31
42
 
32
43
  private
33
44
 
34
45
  def write(chunk)
35
- @compressed_chunk = chunk
46
+ @compressed_chunk << chunk
36
47
  end
37
48
 
38
49
  def compressed_chunk
39
- compressed = @compressed_chunk
40
- compressed
50
+ @compressed_chunk.dup
41
51
  ensure
42
- @compressed_chunk = nil
52
+ @compressed_chunk.clear
43
53
  end
44
54
  end
45
55
 
@@ -17,14 +17,22 @@ module HTTPX
17
17
  def self.extra_options(options)
18
18
  Class.new(options.class) do
19
19
  def_option(:cookies) do |cookies|
20
- return cookies if cookies.is_a?(Store)
21
-
22
- Store.new(cookies)
20
+ if cookies.is_a?(Store)
21
+ cookies
22
+ else
23
+ Store.new(cookies)
24
+ end
23
25
  end
24
26
  end.new(options)
25
27
  end
26
28
 
27
29
  class Store
30
+ def self.new(cookies = nil)
31
+ return cookies if cookies.is_a?(self)
32
+
33
+ super
34
+ end
35
+
28
36
  def initialize(cookies = nil)
29
37
  @store = Hash.new { |hash, origin| hash[origin] = HTTP::CookieJar.new }
30
38
  return unless cookies
@@ -74,10 +82,6 @@ module HTTPX
74
82
  super({ cookies: Store.new }.merge(options), &blk)
75
83
  end
76
84
 
77
- def with_cookies(cookies)
78
- branch(default_options.with_cookies(cookies))
79
- end
80
-
81
85
  def wrap
82
86
  return super unless block_given?
83
87
 
@@ -86,7 +90,7 @@ module HTTPX
86
90
  begin
87
91
  yield session
88
92
  ensure
89
- @options = @options.with_cookies(old_cookies_store)
93
+ @options = @options.with(cookies: old_cookies_store)
90
94
  end
91
95
  end
92
96
  end