httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -57,13 +57,16 @@ module HTTPX
57
57
  def parse_headline
58
58
  idx = @buffer.index("\n")
59
59
  return unless idx
60
+
60
61
  (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
61
62
  raise(Error, "wrong head line format")
62
63
  version, code, _ = m.captures
63
64
  raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
65
+
64
66
  @http_version = version.split(".").map(&:to_i)
65
67
  @status_code = code.to_i
66
68
  raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
69
+
67
70
  @buffer.slice!(0, idx + 1)
68
71
  nextstate(:headers)
69
72
  end
@@ -78,6 +81,7 @@ module HTTPX
78
81
  prepare_data(headers)
79
82
  @observer.on_headers(headers)
80
83
  return unless @state == :headers
84
+
81
85
  # state might have been reset
82
86
  # in the :headers callback
83
87
  nextstate(:data)
@@ -93,12 +97,15 @@ module HTTPX
93
97
  end
94
98
  separator_index = line.index(@header_separator)
95
99
  raise Error, "wrong header format" unless separator_index
100
+
96
101
  key = line[0..separator_index - 1]
97
102
  raise Error, "wrong header format" if key.start_with?("\s", "\t")
103
+
98
104
  key.strip!
99
105
  value = line[separator_index + 1..-1]
100
106
  value.strip!
101
107
  raise Error, "wrong header format" if value.nil?
108
+
102
109
  (headers[key.downcase] ||= []) << value
103
110
  end
104
111
  end
@@ -118,6 +125,7 @@ module HTTPX
118
125
  @buffer.clear
119
126
  end
120
127
  return unless no_more_data?
128
+
121
129
  @buffer = @buffer.to_s
122
130
  if @_has_trailers
123
131
  nextstate(:trailers)
@@ -2,6 +2,10 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds a shim #authentication method to the session, which will fill
7
+ # the HTTP Authorization header.
8
+ #
5
9
  module Authentication
6
10
  module InstanceMethods
7
11
  def authentication(token)
@@ -2,6 +2,10 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds helper methods to implement HTTP Basic Auth
7
+ # https://tools.ietf.org/html/rfc7617
8
+ #
5
9
  module BasicAuthentication
6
10
  def self.load_dependencies(klass, *)
7
11
  require "base64"
@@ -2,23 +2,30 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds compression support. Namely it:
7
+ #
8
+ # * Compresses the request body when passed a supported "Content-Encoding" mime-type;
9
+ # * Decompresses the response body from a supported "Content-Encoding" mime-type;
10
+ #
11
+ # It supports both *gzip* and *deflate*.
12
+ #
5
13
  module Compression
6
14
  extend Registry
7
- def self.configure(klass, *)
15
+ def self.load_dependencies(klass, *)
8
16
  klass.plugin(:"compression/gzip")
9
17
  klass.plugin(:"compression/deflate")
10
18
  end
11
19
 
12
- module InstanceMethods
13
- def initialize(opts = {})
14
- super(opts.merge(headers: { "accept-encoding" => Compression.registry.keys }))
15
- end
20
+ def self.extra_options(options)
21
+ options.merge(headers: { "accept-encoding" => Compression.registry.keys })
16
22
  end
17
23
 
18
24
  module RequestBodyMethods
19
25
  def initialize(*)
20
26
  super
21
27
  return if @body.nil?
28
+
22
29
  @headers.get("content-encoding").each do |encoding|
23
30
  @body = Encoder.new(@body, Compression.registry(encoding).encoder)
24
31
  end
@@ -29,6 +36,9 @@ module HTTPX
29
36
  module ResponseBodyMethods
30
37
  def initialize(*)
31
38
  super
39
+
40
+ return unless @headers.key?("content-encoding")
41
+
32
42
  @_decoders = @headers.get("content-encoding").map do |encoding|
33
43
  Compression.registry(encoding).decoder
34
44
  end
@@ -40,6 +50,8 @@ module HTTPX
40
50
  end
41
51
 
42
52
  def write(chunk)
53
+ return super unless defined?(@_compressed_length)
54
+
43
55
  @_compressed_length -= chunk.bytesize
44
56
  chunk = decompress(chunk)
45
57
  super(chunk)
@@ -47,6 +59,9 @@ module HTTPX
47
59
 
48
60
  def close
49
61
  super
62
+
63
+ return unless defined?(@_decoders)
64
+
50
65
  @_decoders.each(&:close)
51
66
  end
52
67
 
@@ -70,6 +85,7 @@ module HTTPX
70
85
 
71
86
  def each(&blk)
72
87
  return enum_for(__method__) unless block_given?
88
+
73
89
  unless @buffer.size.zero?
74
90
  @buffer.rewind
75
91
  return @buffer.each(&blk)
@@ -97,6 +113,7 @@ module HTTPX
97
113
 
98
114
  def deflate(&blk)
99
115
  return unless @buffer.size.zero?
116
+
100
117
  @body.rewind
101
118
  @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
102
119
  end
@@ -1,60 +1,113 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "forwardable"
4
+
3
5
  module HTTPX
4
6
  module Plugins
7
+ #
8
+ # This plugin implements a persistent cookie jar for the duration of a session.
9
+ #
10
+ # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
11
+ #
5
12
  module Cookies
13
+ using URIExtensions
14
+
15
+ def self.extra_options(options)
16
+ Class.new(options.class) do
17
+ def_option(:cookies) do |cookies|
18
+ return cookies if cookies.is_a?(Store)
19
+
20
+ Store.new(cookies)
21
+ end
22
+ end.new(options)
23
+ end
24
+
25
+ class Store
26
+ def initialize(cookies = nil)
27
+ @store = Hash.new { |hash, origin| hash[origin] = HTTP::CookieJar.new }
28
+ return unless cookies
29
+
30
+ cookies = cookies.split(/ *; */) if cookies.is_a?(String)
31
+ @default_cookies = cookies.map do |cookie, v|
32
+ if cookie.is_a?(HTTP::Cookie)
33
+ cookie
34
+ else
35
+ HTTP::Cookie.new(cookie.to_s, v.to_s)
36
+ end
37
+ end
38
+ end
39
+
40
+ def set(origin, cookies)
41
+ return unless cookies
42
+
43
+ @store[origin].parse(cookies, origin)
44
+ end
45
+
46
+ def [](uri)
47
+ store = @store[uri.origin]
48
+ @default_cookies.each do |cookie|
49
+ c = cookie.dup
50
+ c.domain ||= uri.authority
51
+ c.path ||= uri.path
52
+ store.add(c)
53
+ end if @default_cookies
54
+ store
55
+ end
56
+ end
57
+
6
58
  def self.load_dependencies(*)
7
59
  require "http/cookie"
8
60
  end
9
61
 
10
62
  module InstanceMethods
11
- def cookies(cookies)
12
- branch(default_options.with_cookies(cookies))
63
+ extend Forwardable
64
+
65
+ def_delegator :@options, :cookies
66
+
67
+ def initialize(options = {}, &blk)
68
+ super({ cookies: Store.new }.merge(options), &blk)
13
69
  end
14
- end
15
70
 
16
- module RequestMethods
17
- def initialize(*)
18
- super
19
- @headers.cookies(@options.cookies, self)
71
+ def with_cookies(cookies)
72
+ branch(default_options.with_cookies(cookies))
20
73
  end
21
- end
22
74
 
23
- module HeadersMethods
24
- def cookies(jar, request)
25
- return unless jar
26
- unless jar.is_a?(HTTP::CookieJar)
27
- jar = jar.each_with_object(HTTP::CookieJar.new) do |(k, v), j|
28
- cookie = k.is_a?(HTTP::Cookie) ? v : HTTP::Cookie.new(k.to_s, v.to_s)
29
- cookie.domain = request.authority
30
- cookie.path = request.path
31
- j.add(cookie)
75
+ def wrap
76
+ return super unless block_given?
77
+
78
+ super do |session|
79
+ old_cookies_store = @options.cookies.dup
80
+ begin
81
+ yield session
82
+ ensure
83
+ @options = @options.with_cookies(old_cookies_store)
32
84
  end
33
85
  end
34
- self["cookie"] = HTTP::Cookie.cookie_value(jar.cookies)
35
86
  end
36
- end
37
87
 
38
- module ResponseMethods
39
- def cookie_jar
40
- return @cookie_jar if defined?(@cookie_jar)
41
- return nil unless headers.key?("set-cookie")
42
- @cookie_jar ||= begin
43
- jar = HTTP::CookieJar.new
44
- jar.parse(headers["set-cookie"], @request.uri)
45
- jar
46
- end
88
+ private
89
+
90
+ def on_response(request, response)
91
+ @options.cookies.set(request.origin, response.headers["set-cookie"]) if response.respond_to?(:headers)
92
+
93
+ super
94
+ end
95
+
96
+ def build_request(*, _)
97
+ request = super
98
+ request.headers.set_cookie(@options.cookies[request.uri])
99
+ request
47
100
  end
48
- alias_method :cookies, :cookie_jar
49
101
  end
50
102
 
51
- module OptionsMethods
52
- def self.included(klass)
53
- super
54
- klass.def_option(:cookies) do |cookies|
55
- cookies.split(/ *; */) if cookies.is_a?(String)
56
- cookies
57
- end
103
+ module HeadersMethods
104
+ def set_cookie(jar)
105
+ return unless jar
106
+
107
+ cookie_value = HTTP::Cookie.cookie_value(jar.cookies)
108
+ return if cookie_value.empty?
109
+
110
+ add("cookie", cookie_value)
58
111
  end
59
112
  end
60
113
  end
@@ -2,9 +2,23 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds helper methods to implement HTTP Digest Auth
7
+ # https://tools.ietf.org/html/rfc7616
8
+ #
5
9
  module DigestAuthentication
6
10
  DigestError = Class.new(Error)
7
11
 
12
+ def self.extra_options(options)
13
+ Class.new(options.class) do
14
+ def_option(:digest) do |digest|
15
+ raise Error, ":digest must be a Digest" unless digest.is_a?(Digest)
16
+
17
+ digest
18
+ end
19
+ end.new(options)
20
+ end
21
+
8
22
  def self.load_dependencies(*)
9
23
  require "securerandom"
10
24
  require "digest"
@@ -12,37 +26,41 @@ module HTTPX
12
26
 
13
27
  module InstanceMethods
14
28
  def digest_authentication(user, password)
15
- @_digest = Digest.new(user, password)
16
- self
29
+ branch(default_options.with_digest(Digest.new(user, password)))
17
30
  end
31
+
18
32
  alias_method :digest_auth, :digest_authentication
19
33
 
20
- def request(*args, keep_open: @keep_open, **options)
21
- return super unless @_digest
22
- begin
23
- requests = __build_reqs(*args, **options)
24
- probe_request = requests.first
25
- prev_response = __send_reqs(*probe_request).first
34
+ def request(*args, **options)
35
+ requests = build_requests(*args, options)
36
+ probe_request = requests.first
37
+ digest = probe_request.options.digest
26
38
 
27
- unless prev_response.status == 401
28
- raise Error, "request doesn't require authentication (status: #{prev_response})"
29
- end
39
+ return super unless digest
30
40
 
31
- probe_request.transition(:idle)
32
- responses = []
41
+ prev_response = wrap { send_requests(*probe_request, options).first }
33
42
 
34
- requests.each do |request|
35
- token = @_digest.generate_header(request, prev_response)
36
- request.headers["authorization"] = "Digest #{token}"
37
- response = __send_reqs(*request).first
38
- responses << response
39
- prev_response = response
43
+ raise Error, "request doesn't require authentication (status: #{prev_response.status})" unless prev_response.status == 401
44
+
45
+ probe_request.transition(:idle)
46
+
47
+ responses = []
48
+
49
+ while (request = requests.shift)
50
+ token = digest.generate_header(request, prev_response)
51
+ request.headers["authorization"] = "Digest #{token}"
52
+ response = if requests.empty?
53
+ send_requests(*request, options).first
54
+ else
55
+ wrap { send_requests(*request, options).first }
40
56
  end
41
- return responses.first if responses.size == 1
42
- responses
43
- ensure
44
- close unless keep_open
57
+ responses << response
58
+ prev_response = response
45
59
  end
60
+
61
+ return responses.first if responses.size == 1
62
+
63
+ responses
46
64
  end
47
65
  end
48
66
 
@@ -54,10 +72,10 @@ module HTTPX
54
72
  end
55
73
 
56
74
  def generate_header(request, response, _iis = false)
57
- method = request.verb.to_s.upcase
75
+ meth = request.verb.to_s.upcase
58
76
  www = response.headers["www-authenticate"]
59
77
 
60
- # TODO: assert if auth-type is Digest
78
+ # discard first token, it's Digest
61
79
  auth_info = www[/^(\w+) (.*)/, 2]
62
80
 
63
81
  uri = request.path
@@ -100,7 +118,7 @@ module HTTPX
100
118
  end
101
119
 
102
120
  ha1 = algorithm.hexdigest(a1)
103
- ha2 = algorithm.hexdigest("#{method}:#{uri}")
121
+ ha2 = algorithm.hexdigest("#{meth}:#{uri}")
104
122
  request_digest = [ha1, nonce]
105
123
  request_digest.push(nc, cnonce, qop) if qop
106
124
  request_digest << ha2
@@ -136,6 +154,7 @@ module HTTPX
136
154
  end
137
155
  end
138
156
  end
157
+
139
158
  register_plugin :digest_authentication, DigestAuthentication
140
159
  end
141
160
  end
@@ -3,76 +3,74 @@
3
3
  module HTTPX
4
4
  InsecureRedirectError = Class.new(Error)
5
5
  module Plugins
6
+ #
7
+ # This plugin adds support for following redirect (status 30X) responses.
8
+ #
9
+ # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
10
+ # will return the last redirect response. It will **not** raise an exception.
11
+ #
12
+ # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
13
+ #
6
14
  module FollowRedirects
7
- module InstanceMethods
8
- MAX_REDIRECTS = 3
9
- REDIRECT_STATUS = 300..399
15
+ MAX_REDIRECTS = 3
16
+ REDIRECT_STATUS = (300..399).freeze
10
17
 
11
- def max_redirects(n)
12
- branch(default_options.with_max_redirects(n.to_i))
13
- end
18
+ def self.extra_options(options)
19
+ Class.new(options.class) do
20
+ def_option(:max_redirects) do |num|
21
+ num = Integer(num)
22
+ raise Error, ":max_redirects must be positive" unless num.positive?
14
23
 
15
- def request(*args, **options)
16
- # do not needlessly close channels
17
- keep_open = @keep_open
18
- @keep_open = true
19
-
20
- max_redirects = @options.max_redirects || MAX_REDIRECTS
21
- requests = __build_reqs(*args, **options)
22
- responses = __send_reqs(*requests)
23
-
24
- loop do
25
- redirect_requests = []
26
- indexes = responses.each_with_index.map do |response, index|
27
- next unless REDIRECT_STATUS.include?(response.status)
28
- request = requests[index]
29
- retry_request = __build_redirect_req(request, response, options)
30
- redirect_requests << retry_request
31
- index
32
- end.compact
33
- break if redirect_requests.empty?
34
- break if max_redirects <= 0
35
- max_redirects -= 1
36
-
37
- redirect_responses = __send_reqs(*redirect_requests)
38
- indexes.each_with_index do |index, i2|
39
- requests[index] = redirect_requests[i2]
40
- responses[index] = redirect_responses[i2]
41
- end
24
+ num
42
25
  end
43
26
 
44
- return responses.first if responses.size == 1
45
- responses
46
- ensure
47
- @keep_open = keep_open
27
+ def_option(:follow_insecure_redirects)
28
+ end.new(options)
29
+ end
30
+
31
+ module InstanceMethods
32
+ def max_redirects(n)
33
+ branch(default_options.with_max_redirects(n.to_i))
48
34
  end
49
35
 
50
36
  private
51
37
 
52
- def fetch_response(request)
53
- response = super
54
- if response &&
55
- REDIRECT_STATUS.include?(response.status) &&
56
- !@options.follow_insecure_redirects
57
- redirect_uri = __get_location_from_response(response)
58
- if response.uri.scheme == "https" &&
59
- redirect_uri.scheme == "http"
60
- error = InsecureRedirectError.new(redirect_uri.to_s)
61
- error.set_backtrace(caller)
62
- response = ErrorResponse.new(error, @options)
63
- end
38
+ def fetch_response(request, connections, options)
39
+ redirect_request = request.redirect_request
40
+ response = super(redirect_request, connections, options)
41
+ return unless response
42
+
43
+ max_redirects = redirect_request.max_redirects
44
+
45
+ return response unless REDIRECT_STATUS.include?(response.status)
46
+ return response unless max_redirects.positive?
47
+
48
+ retry_request = build_redirect_request(redirect_request, response, options)
49
+
50
+ request.redirect_request = retry_request
51
+
52
+ if !options.follow_insecure_redirects &&
53
+ response.uri.scheme == "https" &&
54
+ retry_request.uri.scheme == "http"
55
+ error = InsecureRedirectError.new(retry_request.uri.to_s)
56
+ error.set_backtrace(caller)
57
+ return ErrorResponse.new(error, options)
64
58
  end
65
- response
59
+
60
+ connection = find_connection(retry_request, connections, options)
61
+ connection.send(retry_request)
62
+ nil
66
63
  end
67
64
 
68
- def __build_redirect_req(request, response, options)
65
+ def build_redirect_request(request, response, options)
69
66
  redirect_uri = __get_location_from_response(response)
67
+ max_redirects = request.max_redirects
70
68
 
71
- # TODO: integrate cookies in the next request
72
69
  # redirects are **ALWAYS** GET
73
70
  retry_options = options.merge(headers: request.headers,
74
- body: request.body)
75
- __build_req(:get, redirect_uri, retry_options)
71
+ body: request.body,
72
+ max_redirects: max_redirects - 1)
73
+ build_request(:get, redirect_uri, retry_options)
76
74
  end
77
75
 
78
76
  def __get_location_from_response(response)
@@ -82,16 +80,17 @@ module HTTPX
82
80
  end
83
81
  end
84
82
 
85
- module OptionsMethods
83
+ module RequestMethods
86
84
  def self.included(klass)
87
- super
88
- klass.def_option(:max_redirects) do |num|
89
- num = Integer(num)
90
- raise Error, ":max_redirects must be positive" unless num.positive?
91
- num
92
- end
85
+ klass.__send__(:attr_writer, :redirect_request)
86
+ end
87
+
88
+ def redirect_request
89
+ @redirect_request || self
90
+ end
93
91
 
94
- klass.def_option(:follow_insecure_redirects)
92
+ def max_redirects
93
+ @options.max_redirects || MAX_REDIRECTS
95
94
  end
96
95
  end
97
96
  end