httpx 0.7.0 → 0.10.0

data/lib/httpx/plugins/cookies/jar.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Cookies
+    # The Cookie Jar
+    #
+    # It holds a bunch of cookies.
+    class Jar
+      using URIExtensions
+
+      include Enumerable
+
+      def initialize_dup(orig)
+        super
+        @cookies = orig.instance_variable_get(:@cookies).dup
+      end
+
+      def initialize(cookies = nil)
+        @cookies = []
+
+        cookies.each do |elem|
+          cookie = case elem
+                   when Cookie
+                     elem
+                   when Array
+                     Cookie.new(*elem)
+                   else
+                     Cookie.new(elem)
+          end
+
+          @cookies << cookie
+        end if cookies
+      end
+
+      def parse(set_cookie)
+        SetCookieParser.call(set_cookie) do |name, value, attrs|
+          add(Cookie.new(name, value, attrs))
+        end
+      end
+
+      def add(cookie, path = nil)
+        c = cookie.dup
+
+        c.path = path if path && c.path == "/"
+
+        @cookies << c
+      end
+
+      def [](uri)
+        each(uri).sort
+      end
+
+      def each(uri = nil, &blk)
+        return enum_for(__method__, uri) unless block_given?
+
+        return @store.each(&blk) unless uri
+
+        uri = URI(uri)
+
+        now = Time.now
+        tpath = uri.path
+
+        @cookies.delete_if do |cookie|
+          if cookie.expired?(now)
+            true
+          else
+            yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "strscan"
+require "time"
+
+module HTTPX
+  module Plugins::Cookies
+    module SetCookieParser
+      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
+
+      # Whitespace.
+      RE_WSP = /[ \t]+/.freeze
+
+      # A pattern that matches a cookie name or attribute name which may
+      # be empty, capturing trailing whitespace.
+      RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
+
+      RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
+
+      # A pattern that matches the comma in a (typically date) value.
+      RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
+
+      module_function
+
+      def scan_dquoted(scanner)
+        s = +""
+
+        until scanner.eos?
+          break if scanner.skip(/"/)
+
+          if scanner.skip(/\\/)
+            s << scanner.getch
+          elsif scanner.scan(/[^"\\]+/)
+            s << scanner.matched
+          end
+        end
+
+        s
+      end
+
+      def scan_value(scanner, comma_as_separator = false)
+        value = +""
+
+        until scanner.eos?
+          if scanner.scan(/[^,;"]+/)
+            value << scanner.matched
+          elsif scanner.skip(/"/)
+            # RFC 6265 2.2
+            # A cookie-value may be DQUOTE'd.
+            value << scan_dquoted(scanner)
+          elsif scanner.check(/;/)
+            break
+          elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
+            break
+          else
+            value << scanner.getch
+          end
+        end
+
+        value.rstrip!
+        value
+      end
+
+      def scan_name_value(scanner, comma_as_separator = false)
+        name = scanner.scan(RE_NAME)
+        name.rstrip! if name
+
+        if scanner.skip(/=/)
+          value = scan_value(scanner, comma_as_separator)
+        else
+          scan_value(scanner, comma_as_separator)
+          value = nil
+        end
+        [name, value]
+      end
+
+      def call(set_cookie)
+        scanner = StringScanner.new(set_cookie)
+
+        # RFC 6265 4.1.1 & 5.2
+        until scanner.eos?
+          start = scanner.pos
+          len = nil
+
+          scanner.skip(RE_WSP)
+
+          name, value = scan_name_value(scanner, true)
+          value = nil if name.empty?
+
+          attrs = {}
+
+          until scanner.eos?
+            if scanner.skip(/,/)
+              # The comma is used as separator for concatenating multiple
+              # values of a header.
+              len = (scanner.pos - 1) - start
+              break
+            elsif scanner.skip(/;/)
+              scanner.skip(RE_WSP)
+
+              aname, avalue = scan_name_value(scanner, true)
+
+              next if aname.empty? || value.nil?
+
+              aname.downcase!
+
+              case aname
+              when "expires"
+                # RFC 6265 5.2.1
+                (avalue &&= Time.httpdate(avalue)) || next
+              when "max-age"
+                # RFC 6265 5.2.2
+                next unless /\A-?\d+\z/.match?(avalue)
+
+                avalue = Integer(avalue)
+              when "domain"
+                # RFC 6265 5.2.3
+                # An empty value SHOULD be ignored.
+                next if avalue.nil? || avalue.empty?
+              when "path"
+                # RFC 6265 5.2.4
+                # A relative path must be ignored rather than normalizing it
+                # to "/".
+                next unless avalue.start_with?("/")
+              when "secure", "httponly"
+                # RFC 6265 5.2.5, 5.2.6
+                avalue = true
+              end
+              attrs[aname] = avalue
+            end
+          end
+
+          len ||= scanner.pos - start
+
+          next if len > Cookie::MAX_LENGTH
+
+          yield(name, value, attrs) if name && !name.empty? && value
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/digest_authentication.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "digest"
+
 module HTTPX
   module Plugins
     #

data/lib/httpx/plugins/expect.rb

@@ -18,14 +18,24 @@ module HTTPX
 
             seconds
           end
+
+          def_option(:expect_threshold_size) do |bytes|
+            bytes = Integer(bytes)
+            raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
+
+            bytes
+          end
         end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
       end
 
       module RequestBodyMethods
-        def initialize(*)
+        def initialize(*, options)
           super
           return if @body.nil?
 
+          threshold = options.expect_threshold_size
+          return if threshold && !unbounded_body? && @body.bytesize < threshold
+
           @headers["expect"] = "100-continue"
         end
       end
@@ -50,6 +60,7 @@ module HTTPX
           return unless response
 
           if response.status == 417 && request.headers.key?("expect")
+            response.close
             request.headers.delete("expect")
             request.transition(:idle)
             connection = find_connection(request, connections, options)

data/lib/httpx/plugins/follow_redirects.rb

@@ -59,8 +59,26 @@ module HTTPX
             return ErrorResponse.new(request, error, options)
           end
 
-          connection = find_connection(retry_request, connections, options)
-          connection.send(retry_request)
+          retry_after = response.headers["retry-after"]
+
+          if retry_after
+            # Servers send the "Retry-After" header field to indicate how long the
+            # user agent ought to wait before making a follow-up request.
+            # When sent with any 3xx (Redirection) response, Retry-After indicates
+            # the minimum time that the user agent is asked to wait before issuing
+            # the redirected request.
+            #
+            retry_after = Utils.parse_retry_after(retry_after)
+
+            log { "redirecting after #{retry_after} secs..." }
+            pool.after(retry_after) do
+              connection = find_connection(retry_request, connections, options)
+              connection.send(retry_request)
+            end
+          else
+            connection = find_connection(retry_request, connections, options)
+            connection.send(retry_request)
+          end
           nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -73,7 +73,7 @@ module HTTPX
 
           # clean up data left behind in the buffer, if the server started
           # sending frames
-          data = response.to_s
+          data = response.read
           @connection << data
         end
       end

data/lib/httpx/plugins/multipart.rb

@@ -29,14 +29,6 @@ module HTTPX
           def bytesize
             @raw.content_length
           end
-
-          def force_encoding(*args)
-            @raw.to_s.force_encoding(*args)
-          end
-
-          def to_str
-            @raw.to_s
-          end
         end
 
         def encode(form)

data/lib/httpx/plugins/persistent.rb

@@ -19,7 +19,12 @@ module HTTPX
     #
     module Persistent
       def self.load_dependencies(klass)
-        klass.plugin(:retries, max_retries: 1, retry_change_requests: true)
+        max_retries = if klass.default_options.respond_to?(:max_retries)
+          [klass.default_options.max_retries, 1].max
+        else
+          1
+        end
+        klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
       end
 
       def self.extra_options(options)

data/lib/httpx/plugins/proxy.rb

@@ -179,20 +179,9 @@ module HTTPX
           super || @state == :connecting || @state == :connected
         end
 
-        def to_io
-          return super unless @options.proxy
-
-          case @state
-          when :idle
-            transition(:connecting)
-          when :connected
-            transition(:open)
-          end
-          @io.to_io
-        end
-
         def call
           super
+
           return unless @options.proxy
 
           case @state
@@ -210,11 +199,26 @@ module HTTPX
           emit(:close)
         end
 
+        private
+
+        def connect
+          return super unless @options.proxy
+
+          case @state
+          when :idle
+            transition(:connecting)
+          when :connected
+            transition(:open)
+          end
+        end
+
         def transition(nextstate)
           return super unless @options.proxy
 
           case nextstate
           when :closing
+            # this is a hack so that we can use the super method
+            # and it'll thing that the current state is open
             @state = :open if @state == :connecting
           end
           super

data/lib/httpx/plugins/proxy/http.rb

@@ -7,6 +7,10 @@ module HTTPX
     module Proxy
       module HTTP
         module ConnectionMethods
+          def connecting?
+            super || @state == :connecting || @state == :connected
+          end
+
           private
 
           def transition(nextstate)
@@ -34,7 +38,6 @@ module HTTPX
               when :idle
                 @parser = ProxyParser.new(@write_buffer, @options)
                 set_parser_callbacks(@parser)
-                @parser.on(:close) { transition(:closing) }
               end
             end
             super
@@ -48,7 +51,7 @@ module HTTPX
             #
             if req.uri.scheme == "https"
               connect_request = ConnectRequest.new(req.uri, @options)
-
+              @inflight += 1
               parser.send(connect_request)
             else
               transition(:connected)
@@ -56,6 +59,7 @@ module HTTPX
           end
 
           def __http_on_connect(_, response)
+            @inflight -= 1
             if response.status == 200
               req = @pending.first
               request_uri = req.uri
@@ -67,6 +71,7 @@ module HTTPX
               while (req = pending.shift)
                 req.emit(:response, response)
               end
+              reset
             end
           end
         end

data/lib/httpx/plugins/proxy/socks4.rb

@@ -39,7 +39,7 @@ module HTTPX
 
               @parser = nil
             end
-            log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
+            log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
             super
           end
 
@@ -91,6 +91,8 @@ module HTTPX
         end
 
         module Packet
+          using(RegexpExtensions) unless Regexp.method_defined?(:match?)
+
           module_function
 
           def connect(parameters, uri)
@@ -101,7 +103,7 @@ module HTTPX
 
               packet << [ip.to_i].pack("N")
             rescue IPAddr::InvalidAddressError
-              if parameters.uri.scheme =~ /^socks4a?$/
+              if /^socks4a?$/.match?(parameters.uri.scheme)
                 # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
                 ip = IPAddr.new(Resolv.getaddress(uri.host))
                 packet << [ip.to_i].pack("N")