httpx 0.9.0 → 0.11.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +48 -0
  3. data/README.md +13 -3
  4. data/doc/release_notes/0_10_0.md +66 -0
  5. data/doc/release_notes/0_10_1.md +37 -0
  6. data/doc/release_notes/0_10_2.md +5 -0
  7. data/doc/release_notes/0_11_0.md +76 -0
  8. data/doc/release_notes/0_11_1.md +1 -0
  9. data/lib/httpx.rb +2 -0
  10. data/lib/httpx/adapters/datadog.rb +205 -0
  11. data/lib/httpx/adapters/faraday.rb +1 -3
  12. data/lib/httpx/adapters/webmock.rb +123 -0
  13. data/lib/httpx/chainable.rb +10 -9
  14. data/lib/httpx/connection.rb +7 -24
  15. data/lib/httpx/connection/http1.rb +15 -2
  16. data/lib/httpx/connection/http2.rb +15 -16
  17. data/lib/httpx/domain_name.rb +438 -0
  18. data/lib/httpx/errors.rb +4 -1
  19. data/lib/httpx/extensions.rb +21 -1
  20. data/lib/httpx/headers.rb +1 -0
  21. data/lib/httpx/io/ssl.rb +4 -9
  22. data/lib/httpx/io/tcp.rb +6 -5
  23. data/lib/httpx/io/udp.rb +8 -4
  24. data/lib/httpx/options.rb +2 -0
  25. data/lib/httpx/parser/http1.rb +14 -17
  26. data/lib/httpx/plugins/compression.rb +28 -63
  27. data/lib/httpx/plugins/compression/brotli.rb +10 -14
  28. data/lib/httpx/plugins/compression/deflate.rb +7 -6
  29. data/lib/httpx/plugins/compression/gzip.rb +23 -5
  30. data/lib/httpx/plugins/cookies.rb +21 -60
  31. data/lib/httpx/plugins/cookies/cookie.rb +173 -0
  32. data/lib/httpx/plugins/cookies/jar.rb +74 -0
  33. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
  34. data/lib/httpx/plugins/expect.rb +34 -11
  35. data/lib/httpx/plugins/follow_redirects.rb +20 -2
  36. data/lib/httpx/plugins/h2c.rb +1 -1
  37. data/lib/httpx/plugins/multipart.rb +41 -30
  38. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  39. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  40. data/lib/httpx/plugins/multipart/part.rb +34 -0
  41. data/lib/httpx/plugins/persistent.rb +6 -1
  42. data/lib/httpx/plugins/proxy.rb +16 -2
  43. data/lib/httpx/plugins/proxy/socks4.rb +14 -14
  44. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  45. data/lib/httpx/plugins/push_promise.rb +2 -2
  46. data/lib/httpx/plugins/rate_limiter.rb +51 -0
  47. data/lib/httpx/plugins/retries.rb +3 -2
  48. data/lib/httpx/plugins/stream.rb +109 -13
  49. data/lib/httpx/pool.rb +14 -20
  50. data/lib/httpx/request.rb +29 -31
  51. data/lib/httpx/resolver.rb +7 -6
  52. data/lib/httpx/resolver/https.rb +25 -25
  53. data/lib/httpx/resolver/native.rb +29 -22
  54. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  55. data/lib/httpx/resolver/system.rb +3 -3
  56. data/lib/httpx/response.rb +16 -23
  57. data/lib/httpx/selector.rb +11 -17
  58. data/lib/httpx/session.rb +39 -30
  59. data/lib/httpx/transcoder.rb +20 -0
  60. data/lib/httpx/transcoder/chunker.rb +0 -2
  61. data/lib/httpx/transcoder/form.rb +9 -7
  62. data/lib/httpx/transcoder/json.rb +0 -4
  63. data/lib/httpx/utils.rb +45 -0
  64. data/lib/httpx/version.rb +1 -1
  65. data/sig/buffer.rbs +24 -0
  66. data/sig/callbacks.rbs +14 -0
  67. data/sig/chainable.rbs +37 -0
  68. data/sig/connection.rbs +85 -0
  69. data/sig/connection/http1.rbs +66 -0
  70. data/sig/connection/http2.rbs +77 -0
  71. data/sig/domain_name.rbs +17 -0
  72. data/sig/errors.rbs +3 -0
  73. data/sig/headers.rbs +45 -0
  74. data/sig/httpx.rbs +15 -0
  75. data/sig/loggable.rbs +11 -0
  76. data/sig/options.rbs +118 -0
  77. data/sig/parser/http1.rbs +50 -0
  78. data/sig/plugins/authentication.rbs +11 -0
  79. data/sig/plugins/basic_authentication.rbs +13 -0
  80. data/sig/plugins/compression.rbs +55 -0
  81. data/sig/plugins/compression/brotli.rbs +21 -0
  82. data/sig/plugins/compression/deflate.rbs +17 -0
  83. data/sig/plugins/compression/gzip.rbs +29 -0
  84. data/sig/plugins/cookies.rbs +26 -0
  85. data/sig/plugins/cookies/cookie.rbs +50 -0
  86. data/sig/plugins/cookies/jar.rbs +27 -0
  87. data/sig/plugins/digest_authentication.rbs +33 -0
  88. data/sig/plugins/expect.rbs +19 -0
  89. data/sig/plugins/follow_redirects.rbs +37 -0
  90. data/sig/plugins/h2c.rbs +26 -0
  91. data/sig/plugins/multipart.rbs +44 -0
  92. data/sig/plugins/persistent.rbs +17 -0
  93. data/sig/plugins/proxy.rbs +47 -0
  94. data/sig/plugins/proxy/http.rbs +14 -0
  95. data/sig/plugins/proxy/socks4.rbs +33 -0
  96. data/sig/plugins/proxy/socks5.rbs +36 -0
  97. data/sig/plugins/proxy/ssh.rbs +18 -0
  98. data/sig/plugins/push_promise.rbs +22 -0
  99. data/sig/plugins/rate_limiter.rbs +11 -0
  100. data/sig/plugins/retries.rbs +48 -0
  101. data/sig/plugins/stream.rbs +39 -0
  102. data/sig/pool.rbs +36 -0
  103. data/sig/registry.rbs +9 -0
  104. data/sig/request.rbs +61 -0
  105. data/sig/resolver.rbs +26 -0
  106. data/sig/resolver/https.rbs +51 -0
  107. data/sig/resolver/native.rbs +60 -0
  108. data/sig/resolver/resolver_mixin.rbs +27 -0
  109. data/sig/resolver/system.rbs +17 -0
  110. data/sig/response.rbs +87 -0
  111. data/sig/selector.rbs +20 -0
  112. data/sig/session.rbs +49 -0
  113. data/sig/timeout.rbs +29 -0
  114. data/sig/transcoder.rbs +18 -0
  115. data/sig/transcoder/body.rbs +20 -0
  116. data/sig/transcoder/chunker.rbs +32 -0
  117. data/sig/transcoder/form.rbs +22 -0
  118. data/sig/transcoder/json.rbs +16 -0
  119. metadata +99 -59
  120. data/lib/httpx/resolver/options.rb +0 -25
@@ -0,0 +1,142 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+ require "time"
5
+
6
+ module HTTPX
7
+ module Plugins::Cookies
8
+ module SetCookieParser
9
+ using(RegexpExtensions) unless Regexp.method_defined?(:match?)
10
+
11
+ # Whitespace.
12
+ RE_WSP = /[ \t]+/.freeze
13
+
14
+ # A pattern that matches a cookie name or attribute name which may
15
+ # be empty, capturing trailing whitespace.
16
+ RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
17
+
18
+ RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
19
+
20
+ # A pattern that matches the comma in a (typically date) value.
21
+ RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
22
+
23
+ module_function
24
+
25
+ def scan_dquoted(scanner)
26
+ s = +""
27
+
28
+ until scanner.eos?
29
+ break if scanner.skip(/"/)
30
+
31
+ if scanner.skip(/\\/)
32
+ s << scanner.getch
33
+ elsif scanner.scan(/[^"\\]+/)
34
+ s << scanner.matched
35
+ end
36
+ end
37
+
38
+ s
39
+ end
40
+
41
+ def scan_value(scanner, comma_as_separator = false)
42
+ value = +""
43
+
44
+ until scanner.eos?
45
+ if scanner.scan(/[^,;"]+/)
46
+ value << scanner.matched
47
+ elsif scanner.skip(/"/)
48
+ # RFC 6265 2.2
49
+ # A cookie-value may be DQUOTE'd.
50
+ value << scan_dquoted(scanner)
51
+ elsif scanner.check(/;/)
52
+ break
53
+ elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
54
+ break
55
+ else
56
+ value << scanner.getch
57
+ end
58
+ end
59
+
60
+ value.rstrip!
61
+ value
62
+ end
63
+
64
+ def scan_name_value(scanner, comma_as_separator = false)
65
+ name = scanner.scan(RE_NAME)
66
+ name.rstrip! if name
67
+
68
+ if scanner.skip(/=/)
69
+ value = scan_value(scanner, comma_as_separator)
70
+ else
71
+ scan_value(scanner, comma_as_separator)
72
+ value = nil
73
+ end
74
+ [name, value]
75
+ end
76
+
77
+ def call(set_cookie)
78
+ scanner = StringScanner.new(set_cookie)
79
+
80
+ # RFC 6265 4.1.1 & 5.2
81
+ until scanner.eos?
82
+ start = scanner.pos
83
+ len = nil
84
+
85
+ scanner.skip(RE_WSP)
86
+
87
+ name, value = scan_name_value(scanner, true)
88
+ value = nil if name.empty?
89
+
90
+ attrs = {}
91
+
92
+ until scanner.eos?
93
+ if scanner.skip(/,/)
94
+ # The comma is used as separator for concatenating multiple
95
+ # values of a header.
96
+ len = (scanner.pos - 1) - start
97
+ break
98
+ elsif scanner.skip(/;/)
99
+ scanner.skip(RE_WSP)
100
+
101
+ aname, avalue = scan_name_value(scanner, true)
102
+
103
+ next if aname.empty? || value.nil?
104
+
105
+ aname.downcase!
106
+
107
+ case aname
108
+ when "expires"
109
+ # RFC 6265 5.2.1
110
+ (avalue &&= Time.httpdate(avalue)) || next
111
+ when "max-age"
112
+ # RFC 6265 5.2.2
113
+ next unless /\A-?\d+\z/.match?(avalue)
114
+
115
+ avalue = Integer(avalue)
116
+ when "domain"
117
+ # RFC 6265 5.2.3
118
+ # An empty value SHOULD be ignored.
119
+ next if avalue.nil? || avalue.empty?
120
+ when "path"
121
+ # RFC 6265 5.2.4
122
+ # A relative path must be ignored rather than normalizing it
123
+ # to "/".
124
+ next unless avalue.start_with?("/")
125
+ when "secure", "httponly"
126
+ # RFC 6265 5.2.5, 5.2.6
127
+ avalue = true
128
+ end
129
+ attrs[aname] = avalue
130
+ end
131
+ end
132
+
133
+ len ||= scanner.pos - start
134
+
135
+ next if len > Cookie::MAX_LENGTH
136
+
137
+ yield(name, value, attrs) if name && !name.empty? && value
138
+ end
139
+ end
140
+ end
141
+ end
142
+ end
@@ -10,6 +10,10 @@ module HTTPX
10
10
  module Expect
11
11
  EXPECT_TIMEOUT = 2
12
12
 
13
+ def self.no_expect_store
14
+ @no_expect_store ||= []
15
+ end
16
+
13
17
  def self.extra_options(options)
14
18
  Class.new(options.class) do
15
19
  def_option(:expect_timeout) do |seconds|
@@ -28,28 +32,46 @@ module HTTPX
28
32
  end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
29
33
  end
30
34
 
31
- module RequestBodyMethods
32
- def initialize(*, options)
35
+ module RequestMethods
36
+ def initialize(*)
33
37
  super
34
- return if @body.nil?
38
+ return if @body.empty?
35
39
 
36
- if (threshold = options.expect_threshold_size)
37
- unless unbounded_body?
38
- return if @body.bytesize < threshold
39
- end
40
- end
40
+ threshold = @options.expect_threshold_size
41
+ return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
42
+
43
+ return if Expect.no_expect_store.include?(origin)
41
44
 
42
45
  @headers["expect"] = "100-continue"
43
46
  end
47
+
48
+ def response=(response)
49
+ if response && response.status == 100 &&
50
+ !@headers.key?("expect") &&
51
+ (@state == :body || @state == :done)
52
+
53
+ # if we're past this point, this means that we just received a 100-Continue response,
54
+ # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
55
+ #
56
+ # this means that expect was deactivated for this request too soon, i.e. response took longer.
57
+ #
58
+ # so we have to reactivate it again.
59
+ @headers["expect"] = "100-continue"
60
+ @informational_status = 100
61
+ Expect.no_expect_store.delete(origin)
62
+ end
63
+ super
64
+ end
44
65
  end
45
66
 
46
67
  module ConnectionMethods
47
68
  def send(request)
48
- request.once(:expects) do
69
+ request.once(:expect) do
49
70
  @timers.after(@options.expect_timeout) do
50
- if request.state == :expects && !request.expects?
71
+ if request.state == :expect && !request.expects?
72
+ Expect.no_expect_store << request.origin
51
73
  request.headers.delete("expect")
52
- handle(request)
74
+ consume
53
75
  end
54
76
  end
55
77
  end
@@ -63,6 +85,7 @@ module HTTPX
63
85
  return unless response
64
86
 
65
87
  if response.status == 417 && request.headers.key?("expect")
88
+ response.close
66
89
  request.headers.delete("expect")
67
90
  request.transition(:idle)
68
91
  connection = find_connection(request, connections, options)
@@ -59,8 +59,26 @@ module HTTPX
59
59
  return ErrorResponse.new(request, error, options)
60
60
  end
61
61
 
62
- connection = find_connection(retry_request, connections, options)
63
- connection.send(retry_request)
62
+ retry_after = response.headers["retry-after"]
63
+
64
+ if retry_after
65
+ # Servers send the "Retry-After" header field to indicate how long the
66
+ # user agent ought to wait before making a follow-up request.
67
+ # When sent with any 3xx (Redirection) response, Retry-After indicates
68
+ # the minimum time that the user agent is asked to wait before issuing
69
+ # the redirected request.
70
+ #
71
+ retry_after = Utils.parse_retry_after(retry_after)
72
+
73
+ log { "redirecting after #{retry_after} secs..." }
74
+ pool.after(retry_after) do
75
+ connection = find_connection(retry_request, connections, options)
76
+ connection.send(retry_request)
77
+ end
78
+ else
79
+ connection = find_connection(retry_request, connections, options)
80
+ connection.send(retry_request)
81
+ end
64
82
  nil
65
83
  end
66
84
 
@@ -73,7 +73,7 @@ module HTTPX
73
73
 
74
74
  # clean up data left behind in the buffer, if the server started
75
75
  # sending frames
76
- data = response.to_s
76
+ data = response.read
77
77
  @connection << data
78
78
  end
79
79
  end
@@ -10,47 +10,58 @@ module HTTPX
10
10
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Multipart-Uploads
11
11
  #
12
12
  module Multipart
13
- module FormTranscoder
14
- module_function
15
-
16
- class Encoder
17
- extend Forwardable
18
-
19
- def_delegator :@raw, :content_type
20
-
21
- def_delegator :@raw, :to_s
13
+ MULTIPART_VALUE_COND = lambda do |value|
14
+ value.respond_to?(:read) ||
15
+ (value.respond_to?(:to_hash) &&
16
+ value.key?(:body) &&
17
+ (value.key?(:filename) || value.key?(:content_type)))
18
+ end
22
19
 
23
- def_delegator :@raw, :read
20
+ class << self
21
+ def normalize_keys(key, value, &block)
22
+ Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
23
+ end
24
24
 
25
- def initialize(form)
26
- @raw = HTTP::FormData.create(form)
25
+ def load_dependencies(*)
26
+ begin
27
+ unless defined?(HTTP::FormData)
28
+ # in order not to break legacy code, we'll keep loading http/form_data for them.
29
+ require "http/form_data"
30
+ warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
31
+ "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
32
+ "If you'd like to stop seeing this message, require 'http/form_data' yourself."
33
+ end
34
+ rescue LoadError
27
35
  end
36
+ require "httpx/plugins/multipart/encoder"
37
+ require "httpx/plugins/multipart/part"
38
+ require "httpx/plugins/multipart/mime_type_detector"
39
+ end
28
40
 
29
- def bytesize
30
- @raw.content_length
31
- end
41
+ def configure(*)
42
+ Transcoder.register("form", FormTranscoder)
43
+ end
44
+ end
32
45
 
33
- def force_encoding(*args)
34
- @raw.to_s.force_encoding(*args)
35
- end
46
+ module FormTranscoder
47
+ module_function
36
48
 
37
- def to_str
38
- @raw.to_s
49
+ def encode(form)
50
+ if multipart?(form)
51
+ Encoder.new(form)
52
+ else
53
+ Transcoder::Form::Encoder.new(form)
39
54
  end
40
55
  end
41
56
 
42
- def encode(form)
43
- Encoder.new(form)
57
+ def multipart?(data)
58
+ data.any? do |_, v|
59
+ MULTIPART_VALUE_COND.call(v) ||
60
+ (v.respond_to?(:to_ary) && v.to_ary.any?(&MULTIPART_VALUE_COND)) ||
61
+ (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| MULTIPART_VALUE_COND.call(e) })
62
+ end
44
63
  end
45
64
  end
46
-
47
- def self.load_dependencies(*)
48
- require "http/form_data"
49
- end
50
-
51
- def self.configure(*)
52
- Transcoder.register("form", FormTranscoder)
53
- end
54
65
  end
55
66
  register_plugin :multipart, Multipart
56
67
  end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX::Plugins
4
+ module Multipart
5
+ class Encoder
6
+ attr_reader :bytesize
7
+
8
+ def initialize(form)
9
+ @boundary = ("-" * 21) << SecureRandom.hex(21)
10
+ @part_index = 0
11
+ @buffer = "".b
12
+
13
+ @form = form
14
+ @parts = to_parts(form)
15
+ end
16
+
17
+ def content_type
18
+ "multipart/form-data; boundary=#{@boundary}"
19
+ end
20
+
21
+ def read(length = nil, outbuf = nil)
22
+ data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
23
+ data ||= "".b
24
+
25
+ read_chunks(data, length)
26
+
27
+ data unless length && data.empty?
28
+ end
29
+
30
+ def rewind
31
+ form = @form.each_with_object([]) do |(key, val), aux|
32
+ v = case val
33
+ when File
34
+ val = val.reopen(val.path, File::RDONLY) if val.closed?
35
+ val.rewind
36
+ val
37
+ else
38
+ v
39
+ end
40
+ aux << [key, v]
41
+ end
42
+ @form = form
43
+ @parts = to_parts(form)
44
+ end
45
+
46
+ private
47
+
48
+ def to_parts(form)
49
+ @bytesize = 0
50
+ params = form.each_with_object([]) do |(key, val), aux|
51
+ Multipart.normalize_keys(key, val) do |k, v|
52
+ next if v.nil?
53
+
54
+ value, content_type, filename = Part.call(v)
55
+
56
+ header = header_part(k, content_type, filename)
57
+ @bytesize += header.size
58
+ aux << header
59
+
60
+ @bytesize += value.size
61
+ aux << value
62
+
63
+ delimiter = StringIO.new("\r\n")
64
+ @bytesize += delimiter.size
65
+ aux << delimiter
66
+ end
67
+ end
68
+ final_delimiter = StringIO.new("--#{@boundary}--\r\n")
69
+ @bytesize += final_delimiter.size
70
+ params << final_delimiter
71
+
72
+ params
73
+ end
74
+
75
+ def header_part(key, content_type, filename)
76
+ header = "--#{@boundary}\r\n".b
77
+ header << "Content-Disposition: form-data; name=#{key.inspect}".b
78
+ header << "; filename=#{filename.inspect}" if filename
79
+ header << "\r\nContent-Type: #{content_type}\r\n\r\n"
80
+ StringIO.new(header)
81
+ end
82
+
83
+ def read_chunks(buffer, length = nil)
84
+ while @part_index < @parts.size
85
+ chunk = read_from_part(length)
86
+
87
+ next unless chunk
88
+
89
+ buffer << chunk.force_encoding(Encoding::BINARY)
90
+
91
+ next unless length
92
+
93
+ length -= chunk.bytesize
94
+
95
+ break if length.zero?
96
+ end
97
+ end
98
+
99
+ # if there's a current part to read from, tries to read a chunk.
100
+ def read_from_part(max_length = nil)
101
+ part = @parts[@part_index]
102
+
103
+ chunk = part.read(max_length, @buffer)
104
+
105
+ return chunk if chunk && !chunk.empty?
106
+
107
+ part.close if part.respond_to?(:close)
108
+
109
+ @part_index += 1
110
+
111
+ nil
112
+ end
113
+ end
114
+ end
115
+ end