httpx 0.10.0 → 0.11.2

data/lib/httpx/plugins/multipart/encoder.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  module Multipart
+    class Encoder
+      attr_reader :bytesize
+
+      def initialize(form)
+        @boundary = ("-" * 21) << SecureRandom.hex(21)
+        @part_index = 0
+        @buffer = "".b
+
+        @form = form
+        @parts = to_parts(form)
+      end
+
+      def content_type
+        "multipart/form-data; boundary=#{@boundary}"
+      end
+
+      def read(length = nil, outbuf = nil)
+        data   = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+        data ||= "".b
+
+        read_chunks(data, length)
+
+        data unless length && data.empty?
+      end
+
+      def rewind
+        form = @form.each_with_object([]) do |(key, val), aux|
+          v = case val
+              when File
+                val = val.reopen(val.path, File::RDONLY) if val.closed?
+                val.rewind
+                val
+              else
+                v
+          end
+          aux << [key, v]
+        end
+        @form = form
+        @parts = to_parts(form)
+      end
+
+      private
+
+      def to_parts(form)
+        @bytesize = 0
+        params = form.each_with_object([]) do |(key, val), aux|
+          Multipart.normalize_keys(key, val) do |k, v|
+            next if v.nil?
+
+            value, content_type, filename = Part.call(v)
+
+            header = header_part(k, content_type, filename)
+            @bytesize += header.size
+            aux << header
+
+            @bytesize += value.size
+            aux << value
+
+            delimiter = StringIO.new("\r\n")
+            @bytesize += delimiter.size
+            aux << delimiter
+          end
+        end
+        final_delimiter = StringIO.new("--#{@boundary}--\r\n")
+        @bytesize += final_delimiter.size
+        params << final_delimiter
+
+        params
+      end
+
+      def header_part(key, content_type, filename)
+        header = "--#{@boundary}\r\n".b
+        header << "Content-Disposition: form-data; name=#{key.inspect}".b
+        header << "; filename=#{filename.inspect}" if filename
+        header << "\r\nContent-Type: #{content_type}\r\n\r\n"
+        StringIO.new(header)
+      end
+
+      def read_chunks(buffer, length = nil)
+        while @part_index < @parts.size
+          chunk = read_from_part(length)
+
+          next unless chunk
+
+          buffer << chunk.force_encoding(Encoding::BINARY)
+
+          next unless length
+
+          length -= chunk.bytesize
+
+          break if length.zero?
+        end
+      end
+
+      # if there's a current part to read from, tries to read a chunk.
+      def read_from_part(max_length = nil)
+        part = @parts[@part_index]
+
+        chunk = part.read(max_length, @buffer)
+
+        return chunk if chunk && !chunk.empty?
+
+        part.close if part.respond_to?(:close)
+
+        @part_index += 1
+
+        nil
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module MimeTypeDetector
+      module_function
+
+      DEFAULT_MIMETYPE = "application/octet-stream"
+
+      # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
+      if defined?(MIME::Types)
+
+        def call(_file, filename)
+          mime = MIME::Types.of(filename).first
+          mime.content_type if mime
+        end
+
+      elsif defined?(MimeMagic)
+
+        def call(file, *)
+          mime = MimeMagic.by_magic(file)
+          mime.type if mime
+        end
+
+      elsif system("which file", out: File::NULL)
+        require "open3"
+
+        def call(file, *)
+          return if file.eof? # file command returns "application/x-empty" for empty files
+
+          Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
+            begin
+              ::IO.copy_stream(file, stdin.binmode)
+            rescue Errno::EPIPE
+            end
+            file.rewind
+            stdin.close
+
+            status = thread.value
+
+            # call to file command failed
+            if status.nil? || !status.success?
+              $stderr.print(stderr.read)
+            else
+
+              output = stdout.read.strip
+
+              if output.include?("cannot open")
+                $stderr.print(output)
+              else
+                output
+              end
+            end
+          end
+        end
+
+      else
+
+        def call(*); end
+
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/part.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module Part
+      module_function
+
+      def call(value)
+        # take out specialized objects of the way
+        if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
+          return [value, value.content_type, value.filename]
+        end
+
+        content_type = filename = nil
+
+        if value.is_a?(Hash)
+          content_type = value[:content_type]
+          filename = value[:filename]
+          value = value[:body]
+        end
+
+        value = value.open(:binmode => true) if value.is_a?(Pathname)
+
+        if value.is_a?(File)
+          filename ||= File.basename(value.path)
+          content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
+          [value, content_type, filename]
+        else
+          [StringIO.new(value.to_s), "text/plain"]
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/proxy.rb

@@ -121,8 +121,7 @@ module HTTPX
         def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
-             # either it was a timeout error connecting, or it was a proxy error
-             PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
+             __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
 
           super
         end
+
+        def __proxy_error?(response)
+          error = response.error
+          case error
+          when ResolveError
+            # failed resolving proxy domain
+            proxy_uri = error.connection.options.proxy.uri
+            proxy_uri.to_s == @_proxy_uris.first
+          when *PROXY_ERRORS
+            # timeout errors connecting to proxy
+            true
+          else
+            false
+          end
+        end
       end
 
       module ConnectionMethods

data/lib/httpx/plugins/proxy/socks4.rb

@@ -10,7 +10,7 @@ module HTTPX
       module Socks4
         VERSION = 4
         CONNECT = 1
-        GRANTED = 90
+        GRANTED = 0x5A
         PROTOCOLS = %w[socks4 socks4a].freeze
 
         Error = Socks4Error
@@ -91,27 +91,25 @@ module HTTPX
         end
 
         module Packet
-          using(RegexpExtensions) unless Regexp.method_defined?(:match?)
-
           module_function
 
           def connect(parameters, uri)
             packet = [VERSION, CONNECT, uri.port].pack("CCn")
-            begin
-              ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
-
-              packet << [ip.to_i].pack("N")
-            rescue IPAddr::InvalidAddressError
-              if /^socks4a?$/.match?(parameters.uri.scheme)
-                # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
-                ip = IPAddr.new(Resolv.getaddress(uri.host))
-                packet << [ip.to_i].pack("N")
-              else
-                packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
+
+            case parameters.uri.scheme
+            when "socks4"
+              socks_host = uri.host
+              begin
+                ip = IPAddr.new(socks_host)
+                packet << ip.hton
+              rescue IPAddr::InvalidAddressError
+                socks_host = Resolv.getaddress(socks_host)
+                retry
               end
+              packet << [parameters.username].pack("Z*")
+            when "socks4a"
+              packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
             end
-            packet << [parameters.username].pack("Z*")
             packet
           end
         end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -159,9 +159,10 @@ module HTTPX
             packet = [VERSION, CONNECT, 0].pack("C*")
             begin
               ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
 
-              packet << [IPV4, ip.to_i].pack("CN")
+              ipcode = ip.ipv6? ? IPV6 : IPV4
+
+              packet << [ipcode].pack("C") << ip.hton
             rescue IPAddr::InvalidAddressError
               packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
             end

data/lib/httpx/plugins/push_promise.rb

@@ -70,8 +70,8 @@ module HTTPX
           request.transition(:done)
           response = request.response
           response.mark_as_pushed!
-          stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
-          stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
+          stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
+          stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
         end
       end
     end

data/lib/httpx/pool.rb

@@ -108,10 +108,10 @@ module HTTPX
       end
     end
 
-    def on_resolver_error(ch, error)
-      ch.emit(:error, error)
+    def on_resolver_error(connection, error)
+      connection.emit(:error, error)
       # must remove connection by hand, hasn't been started yet
-      unregister_connection(ch)
+      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -144,12 +144,12 @@ module HTTPX
       @connected_connections -= 1
     end
 
-    def coalesce_connections(ch1, ch2)
-      if ch1.coalescable?(ch2)
-        ch1.merge(ch2)
-        @connections.delete(ch2)
+    def coalesce_connections(conn1, conn2)
+      if conn1.coalescable?(conn2)
+        conn1.merge(conn2)
+        @connections.delete(conn2)
       else
-        register_connection(ch2)
+        register_connection(conn2)
       end
     end
 
@@ -168,12 +168,6 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      rescue ArgumentError
-        # this block is here because of an error which happens on CI from time to time
-        warn "tried resolver: #{resolver_type}"
-        warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
-        warn "new: #{resolver_type.method(:new).source_location}"
-        raise
       end
     end
   end

data/lib/httpx/request.rb

@@ -81,6 +81,10 @@ module HTTPX
     def response=(response)
       return unless response
 
+      if response.status == 100
+        @informational_status = response.status
+        return
+      end
       @response = response
     end
 
@@ -106,7 +110,7 @@ module HTTPX
 
       query = []
       if (q = @options.params)
-        query << URI.encode_www_form(q)
+        query << Transcoder.registry("form").encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")
@@ -170,6 +174,12 @@ module HTTPX
         end
       end
 
+      def rewind
+        return if empty?
+
+        @body.rewind if @body.respond_to?(:rewind)
+      end
+
       def empty?
         return true if @body.nil?
         return false if chunked?
@@ -212,6 +222,7 @@ module HTTPX
     def transition(nextstate)
       case nextstate
       when :idle
+        @body.rewind
         @response = nil
         @drainer = nil
       when :headers
@@ -221,15 +232,15 @@ module HTTPX
                       @state == :expect
 
         if @headers.key?("expect")
-          unless @response
-            @state = :expect
-            return
-          end
-
-          case @response.status
-          when 100
-            # deallocate
-            @response = nil
+          if @informational_status && @informational_status == 100
+            # check for 100 Continue response, and deallocate the var
+            # if @informational_status == 100
+            #   @response = nil
+            # end
+          else
+            return if @state == :expect # do not re-set it
+
+            nextstate = :expect
           end
         end
       when :done
@@ -241,8 +252,7 @@ module HTTPX
     end
 
     def expects?
-      @headers["expect"] == "100-continue" &&
-        @response && @response.status == 100
+      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
 
     class ProcIO

data/lib/httpx/resolver.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "httpx/resolver/resolver_mixin"
-require "httpx/resolver/system"
-require "httpx/resolver/native"
-require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
     extend Registry
 
+    RESOLVE_TIMEOUT = 5
+
+    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/system"
+    require "httpx/resolver/native"
+    require "httpx/resolver/https"
+
     register :system, System
     register :native, Native
     register :https,  HTTPS
@@ -101,5 +104,3 @@ module HTTPX
     end
   end
 end
-
-require "httpx/resolver/options"