httpx 0.8.0 → 0.10.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (138) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +9 -5
  4. data/doc/release_notes/0_0_1.md +7 -0
  5. data/doc/release_notes/0_0_2.md +9 -0
  6. data/doc/release_notes/0_0_3.md +9 -0
  7. data/doc/release_notes/0_0_4.md +7 -0
  8. data/doc/release_notes/0_0_5.md +5 -0
  9. data/doc/release_notes/0_10_0.md +66 -0
  10. data/doc/release_notes/0_10_1.md +39 -0
  11. data/doc/release_notes/0_1_0.md +9 -0
  12. data/doc/release_notes/0_2_0.md +5 -0
  13. data/doc/release_notes/0_2_1.md +16 -0
  14. data/doc/release_notes/0_3_0.md +12 -0
  15. data/doc/release_notes/0_3_1.md +6 -0
  16. data/doc/release_notes/0_4_0.md +51 -0
  17. data/doc/release_notes/0_4_1.md +3 -0
  18. data/doc/release_notes/0_5_0.md +15 -0
  19. data/doc/release_notes/0_5_1.md +14 -0
  20. data/doc/release_notes/0_6_0.md +5 -0
  21. data/doc/release_notes/0_6_1.md +6 -0
  22. data/doc/release_notes/0_6_2.md +6 -0
  23. data/doc/release_notes/0_6_3.md +13 -0
  24. data/doc/release_notes/0_6_4.md +21 -0
  25. data/doc/release_notes/0_6_5.md +22 -0
  26. data/doc/release_notes/0_6_6.md +19 -0
  27. data/doc/release_notes/0_6_7.md +5 -0
  28. data/doc/release_notes/0_7_0.md +46 -0
  29. data/doc/release_notes/0_8_0.md +27 -0
  30. data/doc/release_notes/0_8_1.md +8 -0
  31. data/doc/release_notes/0_8_2.md +7 -0
  32. data/doc/release_notes/0_9_0.md +38 -0
  33. data/lib/httpx.rb +2 -0
  34. data/lib/httpx/adapters/faraday.rb +1 -1
  35. data/lib/httpx/chainable.rb +11 -11
  36. data/lib/httpx/connection.rb +23 -31
  37. data/lib/httpx/connection/http1.rb +30 -4
  38. data/lib/httpx/connection/http2.rb +29 -10
  39. data/lib/httpx/domain_name.rb +440 -0
  40. data/lib/httpx/errors.rb +2 -1
  41. data/lib/httpx/extensions.rb +22 -2
  42. data/lib/httpx/headers.rb +2 -2
  43. data/lib/httpx/io/ssl.rb +0 -1
  44. data/lib/httpx/io/tcp.rb +6 -5
  45. data/lib/httpx/io/udp.rb +4 -1
  46. data/lib/httpx/options.rb +5 -1
  47. data/lib/httpx/parser/http1.rb +14 -17
  48. data/lib/httpx/plugins/compression.rb +46 -65
  49. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  50. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  51. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  52. data/lib/httpx/plugins/cookies.rb +21 -60
  53. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  54. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  55. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  56. data/lib/httpx/plugins/expect.rb +12 -1
  57. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  58. data/lib/httpx/plugins/h2c.rb +1 -1
  59. data/lib/httpx/plugins/multipart.rb +12 -6
  60. data/lib/httpx/plugins/persistent.rb +6 -1
  61. data/lib/httpx/plugins/proxy.rb +16 -2
  62. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  63. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  64. data/lib/httpx/plugins/retries.rb +3 -2
  65. data/lib/httpx/plugins/stream.rb +109 -13
  66. data/lib/httpx/pool.rb +14 -17
  67. data/lib/httpx/request.rb +8 -20
  68. data/lib/httpx/resolver.rb +7 -10
  69. data/lib/httpx/resolver/https.rb +22 -24
  70. data/lib/httpx/resolver/native.rb +19 -16
  71. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  72. data/lib/httpx/resolver/system.rb +2 -2
  73. data/lib/httpx/response.rb +16 -25
  74. data/lib/httpx/selector.rb +11 -18
  75. data/lib/httpx/session.rb +40 -26
  76. data/lib/httpx/transcoder.rb +18 -0
  77. data/lib/httpx/transcoder/chunker.rb +0 -2
  78. data/lib/httpx/transcoder/form.rb +9 -7
  79. data/lib/httpx/transcoder/json.rb +0 -4
  80. data/lib/httpx/utils.rb +45 -0
  81. data/lib/httpx/version.rb +1 -1
  82. data/sig/buffer.rbs +24 -0
  83. data/sig/callbacks.rbs +14 -0
  84. data/sig/chainable.rbs +37 -0
  85. data/sig/connection.rbs +85 -0
  86. data/sig/connection/http1.rbs +66 -0
  87. data/sig/connection/http2.rbs +78 -0
  88. data/sig/domain_name.rbs +17 -0
  89. data/sig/errors.rbs +3 -0
  90. data/sig/headers.rbs +42 -0
  91. data/sig/httpx.rbs +15 -0
  92. data/sig/loggable.rbs +11 -0
  93. data/sig/missing.rbs +12 -0
  94. data/sig/options.rbs +118 -0
  95. data/sig/parser/http1.rbs +50 -0
  96. data/sig/plugins/authentication.rbs +11 -0
  97. data/sig/plugins/basic_authentication.rbs +13 -0
  98. data/sig/plugins/compression.rbs +55 -0
  99. data/sig/plugins/compression/brotli.rbs +21 -0
  100. data/sig/plugins/compression/deflate.rbs +17 -0
  101. data/sig/plugins/compression/gzip.rbs +29 -0
  102. data/sig/plugins/cookies.rbs +26 -0
  103. data/sig/plugins/cookies/cookie.rbs +50 -0
  104. data/sig/plugins/cookies/jar.rbs +27 -0
  105. data/sig/plugins/digest_authentication.rbs +33 -0
  106. data/sig/plugins/expect.rbs +19 -0
  107. data/sig/plugins/follow_redirects.rbs +37 -0
  108. data/sig/plugins/h2c.rbs +26 -0
  109. data/sig/plugins/multipart.rbs +21 -0
  110. data/sig/plugins/persistent.rbs +17 -0
  111. data/sig/plugins/proxy.rbs +47 -0
  112. data/sig/plugins/proxy/http.rbs +14 -0
  113. data/sig/plugins/proxy/socks4.rbs +33 -0
  114. data/sig/plugins/proxy/socks5.rbs +36 -0
  115. data/sig/plugins/proxy/ssh.rbs +18 -0
  116. data/sig/plugins/push_promise.rbs +22 -0
  117. data/sig/plugins/rate_limiter.rbs +11 -0
  118. data/sig/plugins/retries.rbs +48 -0
  119. data/sig/plugins/stream.rbs +39 -0
  120. data/sig/pool.rbs +36 -0
  121. data/sig/registry.rbs +9 -0
  122. data/sig/request.rbs +61 -0
  123. data/sig/resolver.rbs +26 -0
  124. data/sig/resolver/https.rbs +49 -0
  125. data/sig/resolver/native.rbs +60 -0
  126. data/sig/resolver/resolver_mixin.rbs +27 -0
  127. data/sig/resolver/system.rbs +17 -0
  128. data/sig/response.rbs +87 -0
  129. data/sig/selector.rbs +20 -0
  130. data/sig/session.rbs +49 -0
  131. data/sig/timeout.rbs +29 -0
  132. data/sig/transcoder.rbs +18 -0
  133. data/sig/transcoder/body.rbs +18 -0
  134. data/sig/transcoder/chunker.rbs +32 -0
  135. data/sig/transcoder/form.rbs +16 -0
  136. data/sig/transcoder/json.rbs +14 -0
  137. metadata +128 -22
  138. data/lib/httpx/resolver/options.rb +0 -25
@@ -28,6 +28,7 @@ module HTTPX
28
28
 
29
29
  NativeResolveError = Class.new(ResolveError) do
30
30
  attr_reader :connection, :host
31
+
31
32
  def initialize(connection, host, message = "Can't resolve #{host}")
32
33
  @connection = connection
33
34
  @host = host
@@ -40,7 +41,7 @@ module HTTPX
40
41
 
41
42
  def initialize(response)
42
43
  @response = response
43
- super("HTTP Error: #{@response.status}")
44
+ super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
44
45
  end
45
46
 
46
47
  def status
@@ -11,7 +11,7 @@ module HTTPX
11
11
  #
12
12
  # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
13
13
  #
14
- module CurryMethods # :nodoc:
14
+ module CurryMethods
15
15
  # Backport for the Method#curry method, which is part of ruby core since 2.2 .
16
16
  #
17
17
  def curry(*args)
@@ -54,11 +54,31 @@ module HTTPX
54
54
  Numeric.__send__(:include, NegMethods)
55
55
  end
56
56
 
57
+ module RegexpExtensions
58
+ # If you wonder why this is there: the oauth feature uses a refinement to enhance the
59
+ # Regexp class locally with #match? , but this is never tested, because ActiveSupport
60
+ # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
61
+ # :nocov:
62
+ refine(Regexp) do
63
+ def match?(*args)
64
+ !match(*args).nil?
65
+ end
66
+ end
67
+ end
68
+
57
69
  module URIExtensions
58
70
  refine URI::Generic do
71
+ def non_ascii_hostname
72
+ @non_ascii_hostname
73
+ end
74
+
75
+ def non_ascii_hostname=(hostname)
76
+ @non_ascii_hostname = hostname
77
+ end
78
+
59
79
  def authority
60
80
  port_string = port == default_port ? nil : ":#{port}"
61
- "#{host}#{port_string}"
81
+ "#{@non_ascii_hostname || host}#{port_string}"
62
82
  end
63
83
 
64
84
  def origin
@@ -2,7 +2,7 @@
2
2
 
3
3
  module HTTPX
4
4
  class Headers
5
- EMPTY = [].freeze # :nodoc:
5
+ EMPTY = [].freeze
6
6
 
7
7
  class << self
8
8
  def new(headers = nil)
@@ -67,7 +67,7 @@ module HTTPX
67
67
  #
68
68
  def [](field)
69
69
  a = @headers[downcased(field)] || return
70
- a.join(",")
70
+ a.join(", ")
71
71
  end
72
72
 
73
73
  # sets +value+ (if not nil) as single value for the +field+ header.
@@ -42,7 +42,6 @@ module HTTPX
42
42
  # allow reconnections
43
43
  # connect only works if initial @io is a socket
44
44
  @io = @io.io if @io.respond_to?(:io)
45
- @negotiated = false
46
45
  end
47
46
 
48
47
  def connected?
@@ -7,11 +7,7 @@ module HTTPX
7
7
  class TCP
8
8
  include Loggable
9
9
 
10
- attr_reader :ip, :port
11
-
12
- attr_reader :addresses
13
-
14
- attr_reader :state
10
+ attr_reader :ip, :port, :addresses, :state
15
11
 
16
12
  alias_method :host, :ip
17
13
 
@@ -86,6 +82,7 @@ module HTTPX
86
82
  # :nocov:
87
83
  def read(size, buffer)
88
84
  @io.read_nonblock(size, buffer)
85
+ log { "READ: #{buffer.bytesize} bytes..." }
89
86
  buffer.bytesize
90
87
  rescue ::IO::WaitReadable
91
88
  buffer.clear
@@ -96,6 +93,7 @@ module HTTPX
96
93
 
97
94
  def write(buffer)
98
95
  siz = @io.write_nonblock(buffer)
96
+ log { "WRITE: #{siz} bytes..." }
99
97
  buffer.shift!(siz)
100
98
  siz
101
99
  rescue ::IO::WaitWritable
@@ -113,6 +111,7 @@ module HTTPX
113
111
  end
114
112
  return if ret.nil?
115
113
 
114
+ log { "READ: #{buffer.bytesize} bytes..." }
116
115
  buffer.bytesize
117
116
  end
118
117
 
@@ -121,6 +120,8 @@ module HTTPX
121
120
  return 0 if siz == :wait_writable
122
121
  return if siz.nil?
123
122
 
123
+ log { "WRITE: #{siz} bytes..." }
124
+
124
125
  buffer.shift!(siz)
125
126
  siz
126
127
  end
@@ -7,11 +7,12 @@ module HTTPX
7
7
  class UDP
8
8
  include Loggable
9
9
 
10
- def initialize(uri, _, _)
10
+ def initialize(uri, _, options)
11
11
  ip = IPAddr.new(uri.host)
12
12
  @host = ip.to_s
13
13
  @port = uri.port
14
14
  @io = UDPSocket.new(ip.family)
15
+ @options = options
15
16
  end
16
17
 
17
18
  def to_io
@@ -40,6 +41,7 @@ module HTTPX
40
41
 
41
42
  def write(buffer)
42
43
  siz = @io.send(buffer, 0, @host, @port)
44
+ log { "WRITE: #{siz} bytes..." }
43
45
  buffer.shift!(siz)
44
46
  siz
45
47
  end
@@ -49,6 +51,7 @@ module HTTPX
49
51
  def read(size, buffer)
50
52
  data, _ = @io.recvfrom_nonblock(size)
51
53
  buffer.replace(data)
54
+ log { "READ: #{buffer.bytesize} bytes..." }
52
55
  buffer.bytesize
53
56
  rescue ::IO::WaitReadable
54
57
  0
@@ -76,6 +76,8 @@ module HTTPX
76
76
 
77
77
  defaults.merge!(options)
78
78
  defaults.each do |(k, v)|
79
+ next if v.nil?
80
+
79
81
  __send__(:"#{k}=", v)
80
82
  end
81
83
  end
@@ -146,8 +148,10 @@ module HTTPX
146
148
  end
147
149
 
148
150
  def merge(other)
149
- h1 = to_hash
150
151
  h2 = other.to_hash
152
+ return self if h2.empty?
153
+
154
+ h1 = to_hash
151
155
 
152
156
  merged = h1.merge(h2) do |k, v1, v2|
153
157
  case k
@@ -9,10 +9,9 @@ module HTTPX
9
9
 
10
10
  attr_reader :status_code, :http_version, :headers
11
11
 
12
- def initialize(observer, header_separator: ":")
12
+ def initialize(observer)
13
13
  @observer = observer
14
14
  @state = :idle
15
- @header_separator = header_separator
16
15
  @buffer = "".b
17
16
  @headers = {}
18
17
  end
@@ -40,25 +39,25 @@ module HTTPX
40
39
  private
41
40
 
42
41
  def parse
43
- state = @state
44
- case @state
45
- when :idle
46
- parse_headline
47
- when :headers
48
- parse_headers
49
- when :trailers
50
- parse_headers
51
- when :data
52
- parse_data
42
+ loop do
43
+ state = @state
44
+ case @state
45
+ when :idle
46
+ parse_headline
47
+ when :headers, :trailers
48
+ parse_headers
49
+ when :data
50
+ parse_data
51
+ end
52
+ return if @buffer.empty? || state == @state
53
53
  end
54
- parse if !@buffer.empty? && state != @state
55
54
  end
56
55
 
57
56
  def parse_headline
58
57
  idx = @buffer.index("\n")
59
58
  return unless idx
60
59
 
61
- (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
60
+ (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
62
61
  raise(Error, "wrong head line format")
63
62
  version, code, _ = m.captures
64
63
  raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
@@ -91,12 +90,10 @@ module HTTPX
91
90
  @observer.on_trailers(headers)
92
91
  headers.clear
93
92
  nextstate(:complete)
94
- else
95
- raise Error, "wrong header format"
96
93
  end
97
94
  return
98
95
  end
99
- separator_index = line.index(@header_separator)
96
+ separator_index = line.index(":")
100
97
  raise Error, "wrong header format" unless separator_index
101
98
 
102
99
  key = line[0..separator_index - 1]
@@ -14,13 +14,23 @@ module HTTPX
14
14
  #
15
15
  module Compression
16
16
  extend Registry
17
- def self.load_dependencies(klass)
18
- klass.plugin(:"compression/gzip")
19
- klass.plugin(:"compression/deflate")
20
- end
21
17
 
22
- def self.extra_options(options)
23
- options.merge(headers: { "accept-encoding" => Compression.registry.keys })
18
+ class << self
19
+ def load_dependencies(klass)
20
+ klass.plugin(:"compression/gzip")
21
+ klass.plugin(:"compression/deflate")
22
+ end
23
+
24
+ def extra_options(options)
25
+ Class.new(options.class) do
26
+ def_option(:compression_threshold_size) do |bytes|
27
+ bytes = Integer(bytes)
28
+ raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
29
+
30
+ bytes
31
+ end
32
+ end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
33
+ end
24
34
  end
25
35
 
26
36
  module RequestMethods
@@ -32,14 +42,17 @@ module HTTPX
32
42
  end
33
43
 
34
44
  module RequestBodyMethods
35
- def initialize(*)
45
+ def initialize(*, options)
36
46
  super
37
47
  return if @body.nil?
38
48
 
49
+ threshold = options.compression_threshold_size
50
+ return if threshold && !unbounded_body? && @body.bytesize < threshold
51
+
39
52
  @headers.get("content-encoding").each do |encoding|
40
53
  next if encoding == "identity"
41
54
 
42
- @body = Encoder.new(@body, Compression.registry(encoding).encoder)
55
+ @body = Encoder.new(@body, Compression.registry(encoding).deflater)
43
56
  end
44
57
  @headers["content-length"] = @body.bytesize unless chunked?
45
58
  end
@@ -55,57 +68,53 @@ module HTTPX
55
68
 
56
69
  return unless @headers.key?("content-encoding")
57
70
 
58
- @_decoders = @headers.get("content-encoding").map do |encoding|
71
+ # remove encodings that we are able to decode
72
+ @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
73
+
74
+ compressed_length = if @headers.key?("content-length")
75
+ @headers["content-length"].to_i
76
+ else
77
+ Float::INFINITY
78
+ end
79
+
80
+ @_inflaters = @headers.get("content-encoding").map do |encoding|
59
81
  next if encoding == "identity"
60
82
 
61
- decoder = Compression.registry(encoding).decoder
83
+ inflater = Compression.registry(encoding).inflater(compressed_length)
62
84
  # do not uncompress if there is no decoder available. In fact, we can't reliably
63
85
  # continue decompressing beyond that, so ignore.
64
- break unless decoder
86
+ break unless inflater
65
87
 
66
88
  @encodings << encoding
67
- decoder
89
+ inflater
68
90
  end.compact
69
91
 
70
- # remove encodings that we are able to decode
71
- @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
72
-
73
- @_compressed_length = if @headers.key?("content-length")
74
- @headers["content-length"].to_i
75
- else
76
- Float::INFINITY
77
- end
92
+ # this can happen if the only declared encoding is "identity"
93
+ remove_instance_variable(:@_inflaters) if @_inflaters.empty?
78
94
  end
79
95
 
80
96
  def write(chunk)
81
- return super unless defined?(@_compressed_length)
97
+ return super unless defined?(@_inflaters)
82
98
 
83
- @_compressed_length -= chunk.bytesize
84
99
  chunk = decompress(chunk)
85
100
  super(chunk)
86
101
  end
87
102
 
88
- def close
89
- super
90
-
91
- return unless defined?(@_decoders)
92
-
93
- @_decoders.each(&:close)
94
- end
95
-
96
103
  private
97
104
 
98
105
  def decompress(buffer)
99
- @_decoders.reverse_each do |decoder|
100
- buffer = decoder.decode(buffer)
101
- buffer << decoder.finish if @_compressed_length <= 0
106
+ @_inflaters.reverse_each do |inflater|
107
+ buffer = inflater.inflate(buffer)
102
108
  end
103
109
  buffer
104
110
  end
105
111
  end
106
112
 
107
113
  class Encoder
114
+ attr_reader :content_type
115
+
108
116
  def initialize(body, deflater)
117
+ @content_type = body.content_type
109
118
  @body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
110
119
  @buffer = StringIO.new("".b, File::RDWR)
111
120
  @deflater = deflater
@@ -114,11 +123,10 @@ module HTTPX
114
123
  def each(&blk)
115
124
  return enum_for(__method__) unless block_given?
116
125
 
117
- unless @buffer.size.zero?
118
- @buffer.rewind
119
- return @buffer.each(&blk)
120
- end
121
- deflate(&blk)
126
+ return deflate(&blk) if @buffer.size.zero?
127
+
128
+ @buffer.rewind
129
+ @buffer.each(&blk)
122
130
  end
123
131
 
124
132
  def bytesize
@@ -126,17 +134,6 @@ module HTTPX
126
134
  @buffer.size
127
135
  end
128
136
 
129
- def to_s
130
- deflate
131
- @buffer.rewind
132
- @buffer.read
133
- end
134
-
135
- def close
136
- @buffer.close
137
- @body.close
138
- end
139
-
140
137
  private
141
138
 
142
139
  def deflate(&blk)
@@ -146,22 +143,6 @@ module HTTPX
146
143
  @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
147
144
  end
148
145
  end
149
-
150
- class Decoder
151
- extend Forwardable
152
-
153
- def_delegator :@inflater, :finish
154
-
155
- def_delegator :@inflater, :close
156
-
157
- def initialize(inflater)
158
- @inflater = inflater
159
- end
160
-
161
- def decode(chunk)
162
- @inflater.inflate(chunk)
163
- end
164
- end
165
146
  end
166
147
  register_plugin :compression, Compression
167
148
  end
@@ -13,7 +13,7 @@ module HTTPX
13
13
  Compression.register "br", self
14
14
  end
15
15
 
16
- module Encoder
16
+ module Deflater
17
17
  module_function
18
18
 
19
19
  def deflate(raw, buffer, chunk_size:)
@@ -25,28 +25,24 @@ module HTTPX
25
25
  end
26
26
  end
27
27
 
28
- module BrotliWrapper
29
- module_function
30
-
31
- def inflate(text)
32
- ::Brotli.inflate(text)
28
+ class Inflater
29
+ def initialize(bytesize)
30
+ @bytesize = bytesize
33
31
  end
34
32
 
35
- def close; end
36
-
37
- def finish
38
- ""
33
+ def inflate(chunk)
34
+ ::Brotli.inflate(chunk)
39
35
  end
40
36
  end
41
37
 
42
38
  module_function
43
39
 
44
- def encoder
45
- Encoder
40
+ def deflater
41
+ Deflater
46
42
  end
47
43
 
48
- def decoder
49
- Decoder.new(BrotliWrapper)
44
+ def inflater(bytesize)
45
+ Inflater.new(bytesize)
50
46
  end
51
47
  end
52
48
  end