httpx 0.8.1 → 0.10.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +9 -5
  4. data/doc/release_notes/0_0_1.md +7 -0
  5. data/doc/release_notes/0_0_2.md +9 -0
  6. data/doc/release_notes/0_0_3.md +9 -0
  7. data/doc/release_notes/0_0_4.md +7 -0
  8. data/doc/release_notes/0_0_5.md +5 -0
  9. data/doc/release_notes/0_10_0.md +66 -0
  10. data/doc/release_notes/0_10_1.md +37 -0
  11. data/doc/release_notes/0_10_2.md +5 -0
  12. data/doc/release_notes/0_1_0.md +9 -0
  13. data/doc/release_notes/0_2_0.md +5 -0
  14. data/doc/release_notes/0_2_1.md +16 -0
  15. data/doc/release_notes/0_3_0.md +12 -0
  16. data/doc/release_notes/0_3_1.md +6 -0
  17. data/doc/release_notes/0_4_0.md +51 -0
  18. data/doc/release_notes/0_4_1.md +3 -0
  19. data/doc/release_notes/0_5_0.md +15 -0
  20. data/doc/release_notes/0_5_1.md +14 -0
  21. data/doc/release_notes/0_6_0.md +5 -0
  22. data/doc/release_notes/0_6_1.md +6 -0
  23. data/doc/release_notes/0_6_2.md +6 -0
  24. data/doc/release_notes/0_6_3.md +13 -0
  25. data/doc/release_notes/0_6_4.md +21 -0
  26. data/doc/release_notes/0_6_5.md +22 -0
  27. data/doc/release_notes/0_6_6.md +19 -0
  28. data/doc/release_notes/0_6_7.md +5 -0
  29. data/doc/release_notes/0_7_0.md +46 -0
  30. data/doc/release_notes/0_8_0.md +27 -0
  31. data/doc/release_notes/0_8_1.md +8 -0
  32. data/doc/release_notes/0_8_2.md +7 -0
  33. data/doc/release_notes/0_9_0.md +38 -0
  34. data/lib/httpx.rb +2 -0
  35. data/lib/httpx/adapters/faraday.rb +1 -1
  36. data/lib/httpx/chainable.rb +11 -11
  37. data/lib/httpx/connection.rb +23 -31
  38. data/lib/httpx/connection/http1.rb +30 -4
  39. data/lib/httpx/connection/http2.rb +26 -9
  40. data/lib/httpx/domain_name.rb +440 -0
  41. data/lib/httpx/errors.rb +2 -1
  42. data/lib/httpx/extensions.rb +22 -2
  43. data/lib/httpx/headers.rb +1 -1
  44. data/lib/httpx/io/ssl.rb +0 -1
  45. data/lib/httpx/io/tcp.rb +6 -5
  46. data/lib/httpx/io/udp.rb +4 -1
  47. data/lib/httpx/options.rb +5 -1
  48. data/lib/httpx/parser/http1.rb +14 -17
  49. data/lib/httpx/plugins/compression.rb +46 -65
  50. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  51. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  52. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  53. data/lib/httpx/plugins/cookies.rb +21 -60
  54. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  55. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  56. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  57. data/lib/httpx/plugins/expect.rb +12 -1
  58. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  59. data/lib/httpx/plugins/h2c.rb +1 -1
  60. data/lib/httpx/plugins/multipart.rb +12 -6
  61. data/lib/httpx/plugins/persistent.rb +6 -1
  62. data/lib/httpx/plugins/proxy.rb +16 -2
  63. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  64. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  65. data/lib/httpx/plugins/retries.rb +3 -2
  66. data/lib/httpx/plugins/stream.rb +109 -13
  67. data/lib/httpx/pool.rb +14 -20
  68. data/lib/httpx/request.rb +8 -20
  69. data/lib/httpx/resolver.rb +7 -10
  70. data/lib/httpx/resolver/https.rb +22 -24
  71. data/lib/httpx/resolver/native.rb +19 -16
  72. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  73. data/lib/httpx/resolver/system.rb +2 -2
  74. data/lib/httpx/response.rb +16 -25
  75. data/lib/httpx/selector.rb +11 -18
  76. data/lib/httpx/session.rb +40 -26
  77. data/lib/httpx/transcoder.rb +18 -0
  78. data/lib/httpx/transcoder/chunker.rb +0 -2
  79. data/lib/httpx/transcoder/form.rb +9 -7
  80. data/lib/httpx/transcoder/json.rb +0 -4
  81. data/lib/httpx/utils.rb +45 -0
  82. data/lib/httpx/version.rb +1 -1
  83. data/sig/buffer.rbs +24 -0
  84. data/sig/callbacks.rbs +14 -0
  85. data/sig/chainable.rbs +37 -0
  86. data/sig/connection.rbs +85 -0
  87. data/sig/connection/http1.rbs +66 -0
  88. data/sig/connection/http2.rbs +78 -0
  89. data/sig/domain_name.rbs +17 -0
  90. data/sig/errors.rbs +3 -0
  91. data/sig/headers.rbs +42 -0
  92. data/sig/httpx.rbs +15 -0
  93. data/sig/loggable.rbs +11 -0
  94. data/sig/missing.rbs +12 -0
  95. data/sig/options.rbs +118 -0
  96. data/sig/parser/http1.rbs +50 -0
  97. data/sig/plugins/authentication.rbs +11 -0
  98. data/sig/plugins/basic_authentication.rbs +13 -0
  99. data/sig/plugins/compression.rbs +55 -0
  100. data/sig/plugins/compression/brotli.rbs +21 -0
  101. data/sig/plugins/compression/deflate.rbs +17 -0
  102. data/sig/plugins/compression/gzip.rbs +29 -0
  103. data/sig/plugins/cookies.rbs +26 -0
  104. data/sig/plugins/cookies/cookie.rbs +50 -0
  105. data/sig/plugins/cookies/jar.rbs +27 -0
  106. data/sig/plugins/digest_authentication.rbs +33 -0
  107. data/sig/plugins/expect.rbs +19 -0
  108. data/sig/plugins/follow_redirects.rbs +37 -0
  109. data/sig/plugins/h2c.rbs +26 -0
  110. data/sig/plugins/multipart.rbs +21 -0
  111. data/sig/plugins/persistent.rbs +17 -0
  112. data/sig/plugins/proxy.rbs +47 -0
  113. data/sig/plugins/proxy/http.rbs +14 -0
  114. data/sig/plugins/proxy/socks4.rbs +33 -0
  115. data/sig/plugins/proxy/socks5.rbs +36 -0
  116. data/sig/plugins/proxy/ssh.rbs +18 -0
  117. data/sig/plugins/push_promise.rbs +22 -0
  118. data/sig/plugins/rate_limiter.rbs +11 -0
  119. data/sig/plugins/retries.rbs +48 -0
  120. data/sig/plugins/stream.rbs +39 -0
  121. data/sig/pool.rbs +36 -0
  122. data/sig/registry.rbs +9 -0
  123. data/sig/request.rbs +61 -0
  124. data/sig/resolver.rbs +26 -0
  125. data/sig/resolver/https.rbs +49 -0
  126. data/sig/resolver/native.rbs +60 -0
  127. data/sig/resolver/resolver_mixin.rbs +27 -0
  128. data/sig/resolver/system.rbs +17 -0
  129. data/sig/response.rbs +87 -0
  130. data/sig/selector.rbs +20 -0
  131. data/sig/session.rbs +49 -0
  132. data/sig/timeout.rbs +29 -0
  133. data/sig/transcoder.rbs +18 -0
  134. data/sig/transcoder/body.rbs +18 -0
  135. data/sig/transcoder/chunker.rbs +32 -0
  136. data/sig/transcoder/form.rbs +16 -0
  137. data/sig/transcoder/json.rbs +14 -0
  138. metadata +130 -22
  139. data/lib/httpx/resolver/options.rb +0 -25
@@ -4,16 +4,17 @@ module HTTPX
4
4
  module Plugins
5
5
  module Compression
6
6
  module Deflate
7
- def self.load_dependencies(*)
7
+ def self.load_dependencies(klass)
8
8
  require "stringio"
9
9
  require "zlib"
10
+ klass.plugin(:"compression/gzip")
10
11
  end
11
12
 
12
13
  def self.configure(*)
13
14
  Compression.register "deflate", self
14
15
  end
15
16
 
16
- module Encoder
17
+ module Deflater
17
18
  module_function
18
19
 
19
20
  def deflate(raw, buffer, chunk_size:)
@@ -36,12 +37,12 @@ module HTTPX
36
37
 
37
38
  module_function
38
39
 
39
- def encoder
40
- Encoder
40
+ def deflater
41
+ Deflater
41
42
  end
42
43
 
43
- def decoder
44
- Decoder.new(Zlib::Inflate.new(32 + Zlib::MAX_WBITS))
44
+ def inflater(bytesize)
45
+ GZIP::Inflater.new(bytesize)
45
46
  end
46
47
  end
47
48
  end
@@ -14,7 +14,7 @@ module HTTPX
14
14
  Compression.register "gzip", self
15
15
  end
16
16
 
17
- class Encoder
17
+ class Deflater
18
18
  def initialize
19
19
  @compressed_chunk = "".b
20
20
  end
@@ -53,14 +53,32 @@ module HTTPX
53
53
  end
54
54
  end
55
55
 
56
+ class Inflater
57
+ def initialize(bytesize)
58
+ @inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
59
+ @bytesize = bytesize
60
+ @buffer = nil
61
+ end
62
+
63
+ def inflate(chunk)
64
+ buffer = @inflater.inflate(chunk)
65
+ @bytesize -= chunk.bytesize
66
+ if @bytesize <= 0
67
+ buffer << @inflater.finish
68
+ @inflater.close
69
+ end
70
+ buffer
71
+ end
72
+ end
73
+
56
74
  module_function
57
75
 
58
- def encoder
59
- Encoder.new
76
+ def deflater
77
+ Deflater.new
60
78
  end
61
79
 
62
- def decoder
63
- Decoder.new(Zlib::Inflate.new(32 + Zlib::MAX_WBITS))
80
+ def inflater(bytesize)
81
+ Inflater.new(bytesize)
64
82
  end
65
83
  end
66
84
  end
@@ -12,93 +12,55 @@ module HTTPX
12
12
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Cookies
13
13
  #
14
14
  module Cookies
15
- using URIExtensions
15
+ def self.load_dependencies(*)
16
+ require "httpx/plugins/cookies/jar"
17
+ require "httpx/plugins/cookies/cookie"
18
+ require "httpx/plugins/cookies/set_cookie_parser"
19
+ end
16
20
 
17
21
  def self.extra_options(options)
18
22
  Class.new(options.class) do
19
23
  def_option(:cookies) do |cookies|
20
- if cookies.is_a?(Store)
24
+ if cookies.is_a?(Jar)
21
25
  cookies
22
26
  else
23
- Store.new(cookies)
27
+ Jar.new(cookies)
24
28
  end
25
29
  end
26
30
  end.new(options)
27
31
  end
28
32
 
29
- class Store
30
- def self.new(cookies = nil)
31
- return cookies if cookies.is_a?(self)
32
-
33
- super
34
- end
35
-
36
- def initialize(cookies = nil)
37
- @store = Hash.new { |hash, origin| hash[origin] = HTTP::CookieJar.new }
38
- return unless cookies
39
-
40
- cookies = cookies.split(/ *; */) if cookies.is_a?(String)
41
- @default_cookies = cookies.map do |cookie, v|
42
- if cookie.is_a?(HTTP::Cookie)
43
- cookie
44
- else
45
- HTTP::Cookie.new(cookie.to_s, v.to_s)
46
- end
47
- end
48
- end
49
-
50
- def set(origin, cookies)
51
- return unless cookies
52
-
53
- @store[origin].parse(cookies, origin)
54
- end
55
-
56
- def [](uri)
57
- store = @store[uri.origin]
58
- @default_cookies.each do |cookie|
59
- c = cookie.dup
60
- c.domain ||= uri.authority
61
- c.path ||= uri.path
62
- store.add(c)
63
- end if @default_cookies
64
- store
65
- end
66
-
67
- def ==(other)
68
- @store == other.instance_variable_get(:@store)
69
- end
70
- end
71
-
72
- def self.load_dependencies(*)
73
- require "http/cookie"
74
- end
75
-
76
33
  module InstanceMethods
77
34
  extend Forwardable
78
35
 
79
36
  def_delegator :@options, :cookies
80
37
 
81
38
  def initialize(options = {}, &blk)
82
- super({ cookies: Store.new }.merge(options), &blk)
39
+ super({ cookies: Jar.new }.merge(options), &blk)
83
40
  end
84
41
 
85
42
  def wrap
86
43
  return super unless block_given?
87
44
 
88
45
  super do |session|
89
- old_cookies_store = @options.cookies.dup
46
+ old_cookies_jar = @options.cookies.dup
90
47
  begin
91
48
  yield session
92
49
  ensure
93
- @options = @options.with(cookies: old_cookies_store)
50
+ @options = @options.merge(cookies: old_cookies_jar)
94
51
  end
95
52
  end
96
53
  end
97
54
 
98
55
  private
99
56
 
100
- def on_response(request, response)
101
- @options.cookies.set(request.origin, response.headers["set-cookie"]) if response.respond_to?(:headers)
57
+ def on_response(reuest, response)
58
+ if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
59
+
60
+ log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
61
+
62
+ @options.cookies.parse(set_cookie)
63
+ end
102
64
 
103
65
  super
104
66
  end
@@ -111,13 +73,12 @@ module HTTPX
111
73
  end
112
74
 
113
75
  module HeadersMethods
114
- def set_cookie(jar)
115
- return unless jar
76
+ def set_cookie(cookies)
77
+ return if cookies.empty?
116
78
 
117
- cookie_value = HTTP::Cookie.cookie_value(jar.cookies)
118
- return if cookie_value.empty?
79
+ header_value = cookies.sort.join("; ")
119
80
 
120
- add("cookie", cookie_value)
81
+ add("cookie", header_value)
121
82
  end
122
83
  end
123
84
  end
@@ -0,0 +1,173 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Cookies
5
+ # The HTTP Cookie.
6
+ #
7
+ # Contains the single cookie info: name, value and attributes.
8
+ class Cookie
9
+ include Comparable
10
+ # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
11
+ # least)
12
+ MAX_LENGTH = 4096
13
+
14
+ attr_reader :domain, :path, :name, :value, :created_at
15
+
16
+ def path=(path)
17
+ path = String(path)
18
+ @path = path.start_with?("/") ? path : "/"
19
+ end
20
+
21
+ # See #domain.
22
+ def domain=(domain)
23
+ domain = String(domain)
24
+
25
+ if domain.start_with?(".")
26
+ @for_domain = true
27
+ domain = domain[1..-1]
28
+ end
29
+
30
+ return if domain.empty?
31
+
32
+ @domain_name = DomainName.new(domain)
33
+ # RFC 6265 5.3 5.
34
+ @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
35
+
36
+ @domain = @domain_name.hostname
37
+ end
38
+
39
+ # Compares the cookie with another. When there are many cookies with
40
+ # the same name for a URL, the value of the smallest must be used.
41
+ def <=>(other)
42
+ # RFC 6265 5.4
43
+ # Precedence: 1. longer path 2. older creation
44
+ (@name <=> other.name).nonzero? ||
45
+ (other.path.length <=> @path.length).nonzero? ||
46
+ (@created_at <=> other.created_at).nonzero? ||
47
+ @value <=> other.value
48
+ end
49
+
50
+ class << self
51
+ def new(cookie, *args)
52
+ return cookie if cookie.is_a?(self)
53
+
54
+ super
55
+ end
56
+
57
+ # Tests if +target_path+ is under +base_path+ as described in RFC
58
+ # 6265 5.1.4. +base_path+ must be an absolute path.
59
+ # +target_path+ may be empty, in which case it is treated as the
60
+ # root path.
61
+ #
62
+ # e.g.
63
+ #
64
+ # path_match?('/admin/', '/admin/index') == true
65
+ # path_match?('/admin/', '/Admin/index') == false
66
+ # path_match?('/admin/', '/admin/') == true
67
+ # path_match?('/admin/', '/admin') == false
68
+ #
69
+ # path_match?('/admin', '/admin') == true
70
+ # path_match?('/admin', '/Admin') == false
71
+ # path_match?('/admin', '/admins') == false
72
+ # path_match?('/admin', '/admin/') == true
73
+ # path_match?('/admin', '/admin/index') == true
74
+ def path_match?(base_path, target_path)
75
+ base_path.start_with?("/") || (return false)
76
+ # RFC 6265 5.1.4
77
+ bsize = base_path.size
78
+ tsize = target_path.size
79
+ return bsize == 1 if tsize.zero? # treat empty target_path as "/"
80
+ return false unless target_path.start_with?(base_path)
81
+ return true if bsize == tsize || base_path.end_with?("/")
82
+
83
+ target_path[bsize] == "/"
84
+ end
85
+ end
86
+
87
+ def initialize(arg, *attrs)
88
+ @created_at = Time.now
89
+
90
+ if attrs.empty?
91
+ attr_hash = Hash.try_convert(arg)
92
+ else
93
+ @name = arg
94
+ @value, attr_hash = attrs
95
+ attr_hash = Hash.try_convert(attr_hash)
96
+ end
97
+
98
+ attr_hash.each do |key, val|
99
+ key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
100
+
101
+ case key
102
+ when :domain, :path
103
+ __send__(:"#{key}=", val)
104
+ else
105
+ instance_variable_set(:"@#{key}", val)
106
+ end
107
+ end if attr_hash
108
+
109
+ @path ||= "/"
110
+ raise ArgumentError, "name must be specified" if @name.nil?
111
+ end
112
+
113
+ def expires
114
+ @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
115
+ end
116
+
117
+ def expired?(time = Time.now)
118
+ return false unless expires
119
+
120
+ expires <= time
121
+ end
122
+
123
+ # Returns a string for use in the Cookie header, i.e. `name=value`
124
+ # or `name="value"`.
125
+ def cookie_value
126
+ "#{@name}=#{Scanner.quote(@value)}"
127
+ end
128
+ alias_method :to_s, :cookie_value
129
+
130
+ # Tests if it is OK to send this cookie to a given `uri`. A
131
+ # RuntimeError is raised if the cookie's domain is unknown.
132
+ def valid_for_uri?(uri)
133
+ uri = URI(uri)
134
+ # RFC 6265 5.4
135
+
136
+ return false if @secure && uri.scheme != "https"
137
+
138
+ acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
139
+ end
140
+
141
+ private
142
+
143
+ # Tests if it is OK to accept this cookie if it is sent from a given
144
+ # URI/URL, `uri`.
145
+ def acceptable_from_uri?(uri)
146
+ uri = URI(uri)
147
+
148
+ host = DomainName.new(uri.host)
149
+
150
+ # RFC 6265 5.3
151
+ if host.hostname == @domain
152
+ true
153
+ elsif @for_domain # !host-only-flag
154
+ host.cookie_domain?(@domain_name)
155
+ else
156
+ @domain.nil?
157
+ end
158
+ end
159
+
160
+ module Scanner
161
+ RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
162
+
163
+ module_function
164
+
165
+ def quote(s)
166
+ return s unless s.match(RE_BAD_CHAR)
167
+
168
+ "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
169
+ end
170
+ end
171
+ end
172
+ end
173
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins::Cookies
5
+ # The Cookie Jar
6
+ #
7
+ # It holds a bunch of cookies.
8
+ class Jar
9
+ using URIExtensions
10
+
11
+ include Enumerable
12
+
13
+ def initialize_dup(orig)
14
+ super
15
+ @cookies = orig.instance_variable_get(:@cookies).dup
16
+ end
17
+
18
+ def initialize(cookies = nil)
19
+ @cookies = []
20
+
21
+ cookies.each do |elem|
22
+ cookie = case elem
23
+ when Cookie
24
+ elem
25
+ when Array
26
+ Cookie.new(*elem)
27
+ else
28
+ Cookie.new(elem)
29
+ end
30
+
31
+ @cookies << cookie
32
+ end if cookies
33
+ end
34
+
35
+ def parse(set_cookie)
36
+ SetCookieParser.call(set_cookie) do |name, value, attrs|
37
+ add(Cookie.new(name, value, attrs))
38
+ end
39
+ end
40
+
41
+ def add(cookie, path = nil)
42
+ c = cookie.dup
43
+
44
+ c.path = path if path && c.path == "/"
45
+
46
+ @cookies << c
47
+ end
48
+
49
+ def [](uri)
50
+ each(uri).sort
51
+ end
52
+
53
+ def each(uri = nil, &blk)
54
+ return enum_for(__method__, uri) unless block_given?
55
+
56
+ return @store.each(&blk) unless uri
57
+
58
+ uri = URI(uri)
59
+
60
+ now = Time.now
61
+ tpath = uri.path
62
+
63
+ @cookies.delete_if do |cookie|
64
+ if cookie.expired?(now)
65
+ true
66
+ else
67
+ yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
68
+ false
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end