httpx 0.9.0 → 0.11.1

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "strscan"
+require "time"
+
+module HTTPX
+  module Plugins::Cookies
+    module SetCookieParser
+      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
+
+      # Whitespace.
+      RE_WSP = /[ \t]+/.freeze
+
+      # A pattern that matches a cookie name or attribute name which may
+      # be empty, capturing trailing whitespace.
+      RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
+
+      RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
+
+      # A pattern that matches the comma in a (typically date) value.
+      RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
+
+      module_function
+
+      def scan_dquoted(scanner)
+        s = +""
+
+        until scanner.eos?
+          break if scanner.skip(/"/)
+
+          if scanner.skip(/\\/)
+            s << scanner.getch
+          elsif scanner.scan(/[^"\\]+/)
+            s << scanner.matched
+          end
+        end
+
+        s
+      end
+
+      def scan_value(scanner, comma_as_separator = false)
+        value = +""
+
+        until scanner.eos?
+          if scanner.scan(/[^,;"]+/)
+            value << scanner.matched
+          elsif scanner.skip(/"/)
+            # RFC 6265 2.2
+            # A cookie-value may be DQUOTE'd.
+            value << scan_dquoted(scanner)
+          elsif scanner.check(/;/)
+            break
+          elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
+            break
+          else
+            value << scanner.getch
+          end
+        end
+
+        value.rstrip!
+        value
+      end
+
+      def scan_name_value(scanner, comma_as_separator = false)
+        name = scanner.scan(RE_NAME)
+        name.rstrip! if name
+
+        if scanner.skip(/=/)
+          value = scan_value(scanner, comma_as_separator)
+        else
+          scan_value(scanner, comma_as_separator)
+          value = nil
+        end
+        [name, value]
+      end
+
+      def call(set_cookie)
+        scanner = StringScanner.new(set_cookie)
+
+        # RFC 6265 4.1.1 & 5.2
+        until scanner.eos?
+          start = scanner.pos
+          len = nil
+
+          scanner.skip(RE_WSP)
+
+          name, value = scan_name_value(scanner, true)
+          value = nil if name.empty?
+
+          attrs = {}
+
+          until scanner.eos?
+            if scanner.skip(/,/)
+              # The comma is used as separator for concatenating multiple
+              # values of a header.
+              len = (scanner.pos - 1) - start
+              break
+            elsif scanner.skip(/;/)
+              scanner.skip(RE_WSP)
+
+              aname, avalue = scan_name_value(scanner, true)
+
+              next if aname.empty? || value.nil?
+
+              aname.downcase!
+
+              case aname
+              when "expires"
+                # RFC 6265 5.2.1
+                (avalue &&= Time.httpdate(avalue)) || next
+              when "max-age"
+                # RFC 6265 5.2.2
+                next unless /\A-?\d+\z/.match?(avalue)
+
+                avalue = Integer(avalue)
+              when "domain"
+                # RFC 6265 5.2.3
+                # An empty value SHOULD be ignored.
+                next if avalue.nil? || avalue.empty?
+              when "path"
+                # RFC 6265 5.2.4
+                # A relative path must be ignored rather than normalizing it
+                # to "/".
+                next unless avalue.start_with?("/")
+              when "secure", "httponly"
+                # RFC 6265 5.2.5, 5.2.6
+                avalue = true
+              end
+              attrs[aname] = avalue
+            end
+          end
+
+          len ||= scanner.pos - start
+
+          next if len > Cookie::MAX_LENGTH
+
+          yield(name, value, attrs) if name && !name.empty? && value
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/expect.rb

@@ -10,6 +10,10 @@ module HTTPX
     module Expect
       EXPECT_TIMEOUT = 2
 
+      def self.no_expect_store
+        @no_expect_store ||= []
+      end
+
       def self.extra_options(options)
         Class.new(options.class) do
           def_option(:expect_timeout) do |seconds|
@@ -28,28 +32,46 @@ module HTTPX
         end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
       end
 
-      module RequestBodyMethods
-        def initialize(*, options)
+      module RequestMethods
+        def initialize(*)
           super
-          return if @body.nil?
+          return if @body.empty?
 
-          if (threshold = options.expect_threshold_size)
-            unless unbounded_body?
-              return if @body.bytesize < threshold
-            end
-          end
+          threshold = @options.expect_threshold_size
+          return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
+
+          return if Expect.no_expect_store.include?(origin)
 
           @headers["expect"] = "100-continue"
         end
+
+        def response=(response)
+          if response && response.status == 100 &&
+             !@headers.key?("expect") &&
+             (@state == :body || @state == :done)
+
+            # if we're past this point, this means that we just received a 100-Continue response,
+            # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
+            #
+            # this means that expect was deactivated for this request too soon, i.e. response took longer.
+            #
+            # so we have to reactivate it again.
+            @headers["expect"] = "100-continue"
+            @informational_status = 100
+            Expect.no_expect_store.delete(origin)
+          end
+          super
+        end
       end
 
       module ConnectionMethods
         def send(request)
-          request.once(:expects) do
+          request.once(:expect) do
             @timers.after(@options.expect_timeout) do
-              if request.state == :expects && !request.expects?
+              if request.state == :expect && !request.expects?
+                Expect.no_expect_store << request.origin
                 request.headers.delete("expect")
-                handle(request)
+                consume
               end
             end
           end
@@ -63,6 +85,7 @@ module HTTPX
           return unless response
 
           if response.status == 417 && request.headers.key?("expect")
+            response.close
             request.headers.delete("expect")
             request.transition(:idle)
             connection = find_connection(request, connections, options)

data/lib/httpx/plugins/follow_redirects.rb

@@ -59,8 +59,26 @@ module HTTPX
             return ErrorResponse.new(request, error, options)
           end
 
-          connection = find_connection(retry_request, connections, options)
-          connection.send(retry_request)
+          retry_after = response.headers["retry-after"]
+
+          if retry_after
+            # Servers send the "Retry-After" header field to indicate how long the
+            # user agent ought to wait before making a follow-up request.
+            # When sent with any 3xx (Redirection) response, Retry-After indicates
+            # the minimum time that the user agent is asked to wait before issuing
+            # the redirected request.
+            #
+            retry_after = Utils.parse_retry_after(retry_after)
+
+            log { "redirecting after #{retry_after} secs..." }
+            pool.after(retry_after) do
+              connection = find_connection(retry_request, connections, options)
+              connection.send(retry_request)
+            end
+          else
+            connection = find_connection(retry_request, connections, options)
+            connection.send(retry_request)
+          end
           nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -73,7 +73,7 @@ module HTTPX
 
           # clean up data left behind in the buffer, if the server started
           # sending frames
-          data = response.to_s
+          data = response.read
           @connection << data
         end
       end

data/lib/httpx/plugins/multipart.rb

@@ -10,47 +10,58 @@ module HTTPX
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Multipart-Uploads
     #
     module Multipart
-      module FormTranscoder
-        module_function
-
-        class Encoder
-          extend Forwardable
-
-          def_delegator :@raw, :content_type
-
-          def_delegator :@raw, :to_s
+      MULTIPART_VALUE_COND = lambda do |value|
+        value.respond_to?(:read) ||
+          (value.respond_to?(:to_hash) &&
+            value.key?(:body) &&
+            (value.key?(:filename) || value.key?(:content_type)))
+      end
 
-          def_delegator :@raw, :read
+      class << self
+        def normalize_keys(key, value, &block)
+          Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
+        end
 
-          def initialize(form)
-            @raw = HTTP::FormData.create(form)
+        def load_dependencies(*)
+          begin
+            unless defined?(HTTP::FormData)
+              # in order not to break legacy code, we'll keep loading http/form_data for them.
+              require "http/form_data"
+              warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
+                "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
+                "If you'd like to stop seeing this message, require 'http/form_data' yourself."
+            end
+          rescue LoadError
           end
+          require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/part"
+          require "httpx/plugins/multipart/mime_type_detector"
+        end
 
-          def bytesize
-            @raw.content_length
-          end
+        def configure(*)
+          Transcoder.register("form", FormTranscoder)
+        end
+      end
 
-          def force_encoding(*args)
-            @raw.to_s.force_encoding(*args)
-          end
+      module FormTranscoder
+        module_function
 
-          def to_str
-            @raw.to_s
+        def encode(form)
+          if multipart?(form)
+            Encoder.new(form)
+          else
+            Transcoder::Form::Encoder.new(form)
           end
         end
 
-        def encode(form)
-          Encoder.new(form)
+        def multipart?(data)
+          data.any? do |_, v|
+            MULTIPART_VALUE_COND.call(v) ||
+              (v.respond_to?(:to_ary) && v.to_ary.any?(&MULTIPART_VALUE_COND)) ||
+              (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| MULTIPART_VALUE_COND.call(e) })
+          end
         end
       end
-
-      def self.load_dependencies(*)
-        require "http/form_data"
-      end
-
-      def self.configure(*)
-        Transcoder.register("form", FormTranscoder)
-      end
     end
     register_plugin :multipart, Multipart
   end

data/lib/httpx/plugins/multipart/encoder.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  module Multipart
+    class Encoder
+      attr_reader :bytesize
+
+      def initialize(form)
+        @boundary = ("-" * 21) << SecureRandom.hex(21)
+        @part_index = 0
+        @buffer = "".b
+
+        @form = form
+        @parts = to_parts(form)
+      end
+
+      def content_type
+        "multipart/form-data; boundary=#{@boundary}"
+      end
+
+      def read(length = nil, outbuf = nil)
+        data   = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+        data ||= "".b
+
+        read_chunks(data, length)
+
+        data unless length && data.empty?
+      end
+
+      def rewind
+        form = @form.each_with_object([]) do |(key, val), aux|
+          v = case val
+              when File
+                val = val.reopen(val.path, File::RDONLY) if val.closed?
+                val.rewind
+                val
+              else
+                v
+          end
+          aux << [key, v]
+        end
+        @form = form
+        @parts = to_parts(form)
+      end
+
+      private
+
+      def to_parts(form)
+        @bytesize = 0
+        params = form.each_with_object([]) do |(key, val), aux|
+          Multipart.normalize_keys(key, val) do |k, v|
+            next if v.nil?
+
+            value, content_type, filename = Part.call(v)
+
+            header = header_part(k, content_type, filename)
+            @bytesize += header.size
+            aux << header
+
+            @bytesize += value.size
+            aux << value
+
+            delimiter = StringIO.new("\r\n")
+            @bytesize += delimiter.size
+            aux << delimiter
+          end
+        end
+        final_delimiter = StringIO.new("--#{@boundary}--\r\n")
+        @bytesize += final_delimiter.size
+        params << final_delimiter
+
+        params
+      end
+
+      def header_part(key, content_type, filename)
+        header = "--#{@boundary}\r\n".b
+        header << "Content-Disposition: form-data; name=#{key.inspect}".b
+        header << "; filename=#{filename.inspect}" if filename
+        header << "\r\nContent-Type: #{content_type}\r\n\r\n"
+        StringIO.new(header)
+      end
+
+      def read_chunks(buffer, length = nil)
+        while @part_index < @parts.size
+          chunk = read_from_part(length)
+
+          next unless chunk
+
+          buffer << chunk.force_encoding(Encoding::BINARY)
+
+          next unless length
+
+          length -= chunk.bytesize
+
+          break if length.zero?
+        end
+      end
+
+      # if there's a current part to read from, tries to read a chunk.
+      def read_from_part(max_length = nil)
+        part = @parts[@part_index]
+
+        chunk = part.read(max_length, @buffer)
+
+        return chunk if chunk && !chunk.empty?
+
+        part.close if part.respond_to?(:close)
+
+        @part_index += 1
+
+        nil
+      end
+    end
+  end
+end