httpx 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +2 -0
  4. data/doc/release_notes/0_10_0.md +66 -0
  5. data/lib/httpx.rb +2 -0
  6. data/lib/httpx/adapters/faraday.rb +1 -1
  7. data/lib/httpx/chainable.rb +2 -2
  8. data/lib/httpx/connection.rb +3 -9
  9. data/lib/httpx/connection/http1.rb +1 -1
  10. data/lib/httpx/domain_name.rb +440 -0
  11. data/lib/httpx/errors.rb +1 -0
  12. data/lib/httpx/extensions.rb +21 -1
  13. data/lib/httpx/io/ssl.rb +0 -1
  14. data/lib/httpx/io/tcp.rb +6 -5
  15. data/lib/httpx/io/udp.rb +4 -1
  16. data/lib/httpx/options.rb +2 -0
  17. data/lib/httpx/parser/http1.rb +14 -17
  18. data/lib/httpx/plugins/compression.rb +28 -63
  19. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  20. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  21. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  22. data/lib/httpx/plugins/cookies.rb +21 -60
  23. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  24. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  25. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  26. data/lib/httpx/plugins/expect.rb +3 -5
  27. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  28. data/lib/httpx/plugins/h2c.rb +1 -1
  29. data/lib/httpx/plugins/multipart.rb +0 -8
  30. data/lib/httpx/plugins/persistent.rb +6 -1
  31. data/lib/httpx/plugins/proxy/socks4.rb +3 -1
  32. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  33. data/lib/httpx/plugins/retries.rb +3 -2
  34. data/lib/httpx/plugins/stream.rb +109 -13
  35. data/lib/httpx/pool.rb +6 -6
  36. data/lib/httpx/request.rb +7 -19
  37. data/lib/httpx/resolver/https.rb +7 -2
  38. data/lib/httpx/resolver/native.rb +7 -3
  39. data/lib/httpx/response.rb +16 -23
  40. data/lib/httpx/selector.rb +2 -4
  41. data/lib/httpx/session.rb +17 -11
  42. data/lib/httpx/transcoder/chunker.rb +0 -2
  43. data/lib/httpx/transcoder/form.rb +0 -6
  44. data/lib/httpx/transcoder/json.rb +0 -4
  45. data/lib/httpx/utils.rb +45 -0
  46. data/lib/httpx/version.rb +1 -1
  47. data/sig/buffer.rbs +24 -0
  48. data/sig/callbacks.rbs +14 -0
  49. data/sig/chainable.rbs +37 -0
  50. data/sig/connection.rbs +2 -0
  51. data/sig/connection/http2.rbs +4 -0
  52. data/sig/domain_name.rbs +17 -0
  53. data/sig/errors.rbs +3 -0
  54. data/sig/headers.rbs +42 -0
  55. data/sig/httpx.rbs +14 -0
  56. data/sig/loggable.rbs +11 -0
  57. data/sig/missing.rbs +12 -0
  58. data/sig/options.rbs +118 -0
  59. data/sig/parser/http1.rbs +50 -0
  60. data/sig/plugins/authentication.rbs +11 -0
  61. data/sig/plugins/basic_authentication.rbs +13 -0
  62. data/sig/plugins/compression.rbs +55 -0
  63. data/sig/plugins/compression/brotli.rbs +21 -0
  64. data/sig/plugins/compression/deflate.rbs +17 -0
  65. data/sig/plugins/compression/gzip.rbs +29 -0
  66. data/sig/plugins/cookies.rbs +26 -0
  67. data/sig/plugins/cookies/cookie.rbs +50 -0
  68. data/sig/plugins/cookies/jar.rbs +27 -0
  69. data/sig/plugins/digest_authentication.rbs +33 -0
  70. data/sig/plugins/expect.rbs +19 -0
  71. data/sig/plugins/follow_redirects.rbs +37 -0
  72. data/sig/plugins/h2c.rbs +26 -0
  73. data/sig/plugins/multipart.rbs +19 -0
  74. data/sig/plugins/persistent.rbs +17 -0
  75. data/sig/plugins/proxy.rbs +47 -0
  76. data/sig/plugins/proxy/http.rbs +14 -0
  77. data/sig/plugins/proxy/socks4.rbs +33 -0
  78. data/sig/plugins/proxy/socks5.rbs +36 -0
  79. data/sig/plugins/proxy/ssh.rbs +18 -0
  80. data/sig/plugins/push_promise.rbs +22 -0
  81. data/sig/plugins/rate_limiter.rbs +11 -0
  82. data/sig/plugins/retries.rbs +48 -0
  83. data/sig/plugins/stream.rbs +39 -0
  84. data/sig/pool.rbs +2 -0
  85. data/sig/registry.rbs +9 -0
  86. data/sig/request.rbs +61 -0
  87. data/sig/response.rbs +87 -0
  88. data/sig/session.rbs +49 -0
  89. data/sig/test.rbs +9 -0
  90. data/sig/timeout.rbs +29 -0
  91. data/sig/transcoder.rbs +16 -0
  92. data/sig/transcoder/body.rbs +18 -0
  93. data/sig/transcoder/chunker.rbs +32 -0
  94. data/sig/transcoder/form.rbs +16 -0
  95. data/sig/transcoder/json.rbs +14 -0
  96. metadata +60 -17
@@ -28,6 +28,7 @@ module HTTPX
28
28
 
29
29
  NativeResolveError = Class.new(ResolveError) do
30
30
  attr_reader :connection, :host
31
+
31
32
  def initialize(connection, host, message = "Can't resolve #{host}")
32
33
  @connection = connection
33
34
  @host = host
@@ -54,11 +54,31 @@ module HTTPX
54
54
  Numeric.__send__(:include, NegMethods)
55
55
  end
56
56
 
57
+ module RegexpExtensions
58
+ # If you wonder why this is there: the oauth feature uses a refinement to enhance the
59
+ # Regexp class locally with #match? , but this is never tested, because ActiveSupport
60
+ # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
61
+ # :nocov:
62
+ refine(Regexp) do
63
+ def match?(*args)
64
+ !match(*args).nil?
65
+ end
66
+ end
67
+ end
68
+
57
69
  module URIExtensions
58
70
  refine URI::Generic do
71
+ def non_ascii_hostname
72
+ @non_ascii_hostname
73
+ end
74
+
75
+ def non_ascii_hostname=(hostname)
76
+ @non_ascii_hostname = hostname
77
+ end
78
+
59
79
  def authority
60
80
  port_string = port == default_port ? nil : ":#{port}"
61
- "#{host}#{port_string}"
81
+ "#{@non_ascii_hostname || host}#{port_string}"
62
82
  end
63
83
 
64
84
  def origin
@@ -42,7 +42,6 @@ module HTTPX
42
42
  # allow reconnections
43
43
  # connect only works if initial @io is a socket
44
44
  @io = @io.io if @io.respond_to?(:io)
45
- @negotiated = false
46
45
  end
47
46
 
48
47
  def connected?
@@ -7,11 +7,7 @@ module HTTPX
7
7
  class TCP
8
8
  include Loggable
9
9
 
10
- attr_reader :ip, :port
11
-
12
- attr_reader :addresses
13
-
14
- attr_reader :state
10
+ attr_reader :ip, :port, :addresses, :state
15
11
 
16
12
  alias_method :host, :ip
17
13
 
@@ -86,6 +82,7 @@ module HTTPX
86
82
  # :nocov:
87
83
  def read(size, buffer)
88
84
  @io.read_nonblock(size, buffer)
85
+ log { "READ: #{buffer.bytesize} bytes..." }
89
86
  buffer.bytesize
90
87
  rescue ::IO::WaitReadable
91
88
  buffer.clear
@@ -96,6 +93,7 @@ module HTTPX
96
93
 
97
94
  def write(buffer)
98
95
  siz = @io.write_nonblock(buffer)
96
+ log { "WRITE: #{siz} bytes..." }
99
97
  buffer.shift!(siz)
100
98
  siz
101
99
  rescue ::IO::WaitWritable
@@ -113,6 +111,7 @@ module HTTPX
113
111
  end
114
112
  return if ret.nil?
115
113
 
114
+ log { "READ: #{buffer.bytesize} bytes..." }
116
115
  buffer.bytesize
117
116
  end
118
117
 
@@ -121,6 +120,8 @@ module HTTPX
121
120
  return 0 if siz == :wait_writable
122
121
  return if siz.nil?
123
122
 
123
+ log { "WRITE: #{siz} bytes..." }
124
+
124
125
  buffer.shift!(siz)
125
126
  siz
126
127
  end
@@ -7,11 +7,12 @@ module HTTPX
7
7
  class UDP
8
8
  include Loggable
9
9
 
10
- def initialize(uri, _, _)
10
+ def initialize(uri, _, options)
11
11
  ip = IPAddr.new(uri.host)
12
12
  @host = ip.to_s
13
13
  @port = uri.port
14
14
  @io = UDPSocket.new(ip.family)
15
+ @options = options
15
16
  end
16
17
 
17
18
  def to_io
@@ -40,6 +41,7 @@ module HTTPX
40
41
 
41
42
  def write(buffer)
42
43
  siz = @io.send(buffer, 0, @host, @port)
44
+ log { "WRITE: #{siz} bytes..." }
43
45
  buffer.shift!(siz)
44
46
  siz
45
47
  end
@@ -49,6 +51,7 @@ module HTTPX
49
51
  def read(size, buffer)
50
52
  data, _ = @io.recvfrom_nonblock(size)
51
53
  buffer.replace(data)
54
+ log { "READ: #{buffer.bytesize} bytes..." }
52
55
  buffer.bytesize
53
56
  rescue ::IO::WaitReadable
54
57
  0
@@ -76,6 +76,8 @@ module HTTPX
76
76
 
77
77
  defaults.merge!(options)
78
78
  defaults.each do |(k, v)|
79
+ next if v.nil?
80
+
79
81
  __send__(:"#{k}=", v)
80
82
  end
81
83
  end
@@ -9,10 +9,9 @@ module HTTPX
9
9
 
10
10
  attr_reader :status_code, :http_version, :headers
11
11
 
12
- def initialize(observer, header_separator: ":")
12
+ def initialize(observer)
13
13
  @observer = observer
14
14
  @state = :idle
15
- @header_separator = header_separator
16
15
  @buffer = "".b
17
16
  @headers = {}
18
17
  end
@@ -40,25 +39,25 @@ module HTTPX
40
39
  private
41
40
 
42
41
  def parse
43
- state = @state
44
- case @state
45
- when :idle
46
- parse_headline
47
- when :headers
48
- parse_headers
49
- when :trailers
50
- parse_headers
51
- when :data
52
- parse_data
42
+ loop do
43
+ state = @state
44
+ case @state
45
+ when :idle
46
+ parse_headline
47
+ when :headers, :trailers
48
+ parse_headers
49
+ when :data
50
+ parse_data
51
+ end
52
+ return if @buffer.empty? || state == @state
53
53
  end
54
- parse if !@buffer.empty? && state != @state
55
54
  end
56
55
 
57
56
  def parse_headline
58
57
  idx = @buffer.index("\n")
59
58
  return unless idx
60
59
 
61
- (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
60
+ (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
62
61
  raise(Error, "wrong head line format")
63
62
  version, code, _ = m.captures
64
63
  raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
@@ -91,12 +90,10 @@ module HTTPX
91
90
  @observer.on_trailers(headers)
92
91
  headers.clear
93
92
  nextstate(:complete)
94
- else
95
- raise Error, "wrong header format"
96
93
  end
97
94
  return
98
95
  end
99
- separator_index = line.index(@header_separator)
96
+ separator_index = line.index(":")
100
97
  raise Error, "wrong header format" unless separator_index
101
98
 
102
99
  key = line[0..separator_index - 1]
@@ -46,16 +46,13 @@ module HTTPX
46
46
  super
47
47
  return if @body.nil?
48
48
 
49
- if (threshold = options.compression_threshold_size)
50
- unless unbounded_body?
51
- return if @body.bytesize < threshold
52
- end
53
- end
49
+ threshold = options.compression_threshold_size
50
+ return if threshold && !unbounded_body? && @body.bytesize < threshold
54
51
 
55
52
  @headers.get("content-encoding").each do |encoding|
56
53
  next if encoding == "identity"
57
54
 
58
- @body = Encoder.new(@body, Compression.registry(encoding).encoder)
55
+ @body = Encoder.new(@body, Compression.registry(encoding).deflater)
59
56
  end
60
57
  @headers["content-length"] = @body.bytesize unless chunked?
61
58
  end
@@ -71,57 +68,53 @@ module HTTPX
71
68
 
72
69
  return unless @headers.key?("content-encoding")
73
70
 
74
- @_decoders = @headers.get("content-encoding").map do |encoding|
71
+ # remove encodings that we are able to decode
72
+ @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
73
+
74
+ compressed_length = if @headers.key?("content-length")
75
+ @headers["content-length"].to_i
76
+ else
77
+ Float::INFINITY
78
+ end
79
+
80
+ @_inflaters = @headers.get("content-encoding").map do |encoding|
75
81
  next if encoding == "identity"
76
82
 
77
- decoder = Compression.registry(encoding).decoder
83
+ inflater = Compression.registry(encoding).inflater(compressed_length)
78
84
  # do not uncompress if there is no decoder available. In fact, we can't reliably
79
85
  # continue decompressing beyond that, so ignore.
80
- break unless decoder
86
+ break unless inflater
81
87
 
82
88
  @encodings << encoding
83
- decoder
89
+ inflater
84
90
  end.compact
85
91
 
86
- # remove encodings that we are able to decode
87
- @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
88
-
89
- @_compressed_length = if @headers.key?("content-length")
90
- @headers["content-length"].to_i
91
- else
92
- Float::INFINITY
93
- end
92
+ # this can happen if the only declared encoding is "identity"
93
+ remove_instance_variable(:@_inflaters) if @_inflaters.empty?
94
94
  end
95
95
 
96
96
  def write(chunk)
97
- return super unless defined?(@_compressed_length)
97
+ return super unless defined?(@_inflaters)
98
98
 
99
- @_compressed_length -= chunk.bytesize
100
99
  chunk = decompress(chunk)
101
100
  super(chunk)
102
101
  end
103
102
 
104
- def close
105
- super
106
-
107
- return unless defined?(@_decoders)
108
-
109
- @_decoders.each(&:close)
110
- end
111
-
112
103
  private
113
104
 
114
105
  def decompress(buffer)
115
- @_decoders.reverse_each do |decoder|
116
- buffer = decoder.decode(buffer)
117
- buffer << decoder.finish if @_compressed_length <= 0
106
+ @_inflaters.reverse_each do |inflater|
107
+ buffer = inflater.inflate(buffer)
118
108
  end
119
109
  buffer
120
110
  end
121
111
  end
122
112
 
123
113
  class Encoder
114
+ attr_reader :content_type
115
+
124
116
  def initialize(body, deflater)
117
+ @content_type = body.content_type
125
118
  @body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
126
119
  @buffer = StringIO.new("".b, File::RDWR)
127
120
  @deflater = deflater
@@ -130,11 +123,10 @@ module HTTPX
130
123
  def each(&blk)
131
124
  return enum_for(__method__) unless block_given?
132
125
 
133
- unless @buffer.size.zero?
134
- @buffer.rewind
135
- return @buffer.each(&blk)
136
- end
137
- deflate(&blk)
126
+ return deflate(&blk) if @buffer.size.zero?
127
+
128
+ @buffer.rewind
129
+ @buffer.each(&blk)
138
130
  end
139
131
 
140
132
  def bytesize
@@ -142,17 +134,6 @@ module HTTPX
142
134
  @buffer.size
143
135
  end
144
136
 
145
- def to_s
146
- deflate
147
- @buffer.rewind
148
- @buffer.read
149
- end
150
-
151
- def close
152
- @buffer.close
153
- @body.close
154
- end
155
-
156
137
  private
157
138
 
158
139
  def deflate(&blk)
@@ -162,22 +143,6 @@ module HTTPX
162
143
  @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
163
144
  end
164
145
  end
165
-
166
- class Decoder
167
- extend Forwardable
168
-
169
- def_delegator :@inflater, :finish
170
-
171
- def_delegator :@inflater, :close
172
-
173
- def initialize(inflater)
174
- @inflater = inflater
175
- end
176
-
177
- def decode(chunk)
178
- @inflater.inflate(chunk)
179
- end
180
- end
181
146
  end
182
147
  register_plugin :compression, Compression
183
148
  end
@@ -13,7 +13,7 @@ module HTTPX
13
13
  Compression.register "br", self
14
14
  end
15
15
 
16
- module Encoder
16
+ module Deflater
17
17
  module_function
18
18
 
19
19
  def deflate(raw, buffer, chunk_size:)
@@ -25,28 +25,24 @@ module HTTPX
25
25
  end
26
26
  end
27
27
 
28
- module BrotliWrapper
29
- module_function
30
-
31
- def inflate(text)
32
- ::Brotli.inflate(text)
28
+ class Inflater
29
+ def initialize(bytesize)
30
+ @bytesize = bytesize
33
31
  end
34
32
 
35
- def close; end
36
-
37
- def finish
38
- ""
33
+ def inflate(chunk)
34
+ ::Brotli.inflate(chunk)
39
35
  end
40
36
  end
41
37
 
42
38
  module_function
43
39
 
44
- def encoder
45
- Encoder
40
+ def deflater
41
+ Deflater
46
42
  end
47
43
 
48
- def decoder
49
- Decoder.new(BrotliWrapper)
44
+ def inflater(bytesize)
45
+ Inflater.new(bytesize)
50
46
  end
51
47
  end
52
48
  end
@@ -4,16 +4,17 @@ module HTTPX
4
4
  module Plugins
5
5
  module Compression
6
6
  module Deflate
7
- def self.load_dependencies(*)
7
+ def self.load_dependencies(klass)
8
8
  require "stringio"
9
9
  require "zlib"
10
+ klass.plugin(:"compression/gzip")
10
11
  end
11
12
 
12
13
  def self.configure(*)
13
14
  Compression.register "deflate", self
14
15
  end
15
16
 
16
- module Encoder
17
+ module Deflater
17
18
  module_function
18
19
 
19
20
  def deflate(raw, buffer, chunk_size:)
@@ -36,12 +37,12 @@ module HTTPX
36
37
 
37
38
  module_function
38
39
 
39
- def encoder
40
- Encoder
40
+ def deflater
41
+ Deflater
41
42
  end
42
43
 
43
- def decoder
44
- Decoder.new(Zlib::Inflate.new(32 + Zlib::MAX_WBITS))
44
+ def inflater(bytesize)
45
+ GZIP::Inflater.new(bytesize)
45
46
  end
46
47
  end
47
48
  end
@@ -14,7 +14,7 @@ module HTTPX
14
14
  Compression.register "gzip", self
15
15
  end
16
16
 
17
- class Encoder
17
+ class Deflater
18
18
  def initialize
19
19
  @compressed_chunk = "".b
20
20
  end
@@ -53,14 +53,32 @@ module HTTPX
53
53
  end
54
54
  end
55
55
 
56
+ class Inflater
57
+ def initialize(bytesize)
58
+ @inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
59
+ @bytesize = bytesize
60
+ @buffer = nil
61
+ end
62
+
63
+ def inflate(chunk)
64
+ buffer = @inflater.inflate(chunk)
65
+ @bytesize -= chunk.bytesize
66
+ if @bytesize <= 0
67
+ buffer << @inflater.finish
68
+ @inflater.close
69
+ end
70
+ buffer
71
+ end
72
+ end
73
+
56
74
  module_function
57
75
 
58
- def encoder
59
- Encoder.new
76
+ def deflater
77
+ Deflater.new
60
78
  end
61
79
 
62
- def decoder
63
- Decoder.new(Zlib::Inflate.new(32 + Zlib::MAX_WBITS))
80
+ def inflater(bytesize)
81
+ Inflater.new(bytesize)
64
82
  end
65
83
  end
66
84
  end