fog-aliyun 0.3.17 → 0.4.0

data/lib/fog/aliyun/models/storage/file.rb

@@ -7,77 +7,123 @@ module Fog
     class Storage
       class File < Fog::Model
         identity :key, aliases: ['Key', 'Name', 'name']
+
+        attr_writer :body
+        attribute :cache_control, aliases: 'Cache-Control'
+        attribute :content_encoding, aliases: 'Content-Encoding'
         attribute :date, aliases: 'Date'
-        attribute :content_length, aliases: 'Content-Length', type: :integer
+        attribute :content_length, aliases: ['Content-Length', 'Size'], type: :integer
+        attribute :content_md5, aliases: 'Content-MD5'
         attribute :content_type, aliases: 'Content-Type'
         attribute :connection, aliases: 'Connection'
         attribute :content_disposition, aliases: 'Content-Disposition'
-        attribute :etag, aliases: 'Etag'
+        attribute :etag, aliases: ['Etag', 'ETag']
+        attribute :expires, aliases: 'Expires'
+        attribute :metadata
+        attribute :owner, aliases: 'Owner'
         attribute :last_modified, aliases: 'Last-Modified', type: :time
         attribute :accept_ranges, aliases: 'Accept-Ranges'
         attribute :server, aliases: 'Server'
-        attribute :object_type, aliases: 'x-oss-object-type'
+        attribute :object_type, aliases: ['x-oss-object-type', 'x_oss_object_type']
+
+        # @note Chunk size to use for multipart uploads.
+        #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
+        attr_reader :multipart_chunk_size
+        def multipart_chunk_size=(mp_chunk_size)
+          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          @multipart_chunk_size = mp_chunk_size
+        end
+
+        def acl
+          requires :directory, :key
+          service.get_object_acl(directory.key, key)
+        end
+
+        def acl=(new_acl)
+          valid_acls = ['private', 'public-read', 'public-read-write', 'default']
+          unless valid_acls.include?(new_acl)
+            raise ArgumentError.new("acl must be one of [#{valid_acls.join(', ')}]")
+          end
+          @acl = new_acl
+        end
 
         def body
-          attributes[:body] ||=
-            if last_modified
-              collection.get(identity).body
-            else
-              ''
-            end
+          return attributes[:body] if attributes[:body]
+          return '' unless last_modified
+
+          file = collection.get(identity)
+          if file
+            attributes[:body] = file.body
+          else
+            attributes[:body] = ''
+          end
         end
 
         def body=(new_body)
           attributes[:body] = new_body
         end
 
-        attr_reader :directory
+        def directory
+          @directory
+        end
 
+        # Copy object from one bucket to other bucket.
+        #
+        #     required attributes: directory, key
+        #
+        # @param target_directory_key [String]
+        # @param target_file_key [String]
+        # @param options [Hash] options for copy_object method
+        # @return [String] Fog::Aliyun::Files#head status of directory contents
+        #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          source_bucket, directory_key = collection.check_directory_key(directory.key)
-          source_object = if directory_key == ''
-                            key
-                          else
-                            directory_key + '/' + key
-                          end
-          target_bucket, target_directory_key = collection.check_directory_key(target_directory_key)
-          target_object = if target_directory_key == ''
-                            target_file_key
-                          else
-                            target_directory_key + '/' + target_file_key
-                          end
-          service.copy_object(source_bucket, source_object, target_bucket, target_object, options)
-          target_directory = service.directories.new(key: target_directory_key)
-          target_directory.files.get(target_file_key)
-        end
-
-        def destroy
+          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          target_directory = service.directories.new(:key => target_directory_key)
+          target_directory.files.head(target_file_key)
+        end
+
+        def destroy(options = {})
           requires :directory, :key
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          service.delete_object(object, bucket: bucket_name)
+          # TODO support versionId
+          # attributes[:body] = nil if options['versionId'] == version
+          service.delete_object(directory.key, key, options)
           true
         end
 
+        remove_method :metadata
         def metadata
-          attributes[:metadata] ||= headers_to_metadata
+          attributes.reject {|key, value| !(key.to_s =~ /^x-oss-/)}
+        end
+
+        remove_method :metadata=
+        def metadata=(new_metadata)
+          merge_attributes(new_metadata)
         end
 
+        remove_method :owner=
         def owner=(new_owner)
           if new_owner
             attributes[:owner] = {
-              display_name: new_owner['DisplayName'],
-              id: new_owner['ID']
+                :display_name => new_owner['DisplayName'] || new_owner[:display_name],
+                :id           => new_owner['ID'] || new_owner[:id]
             }
           end
         end
 
+        # Set Access-Control-List permissions.
+        #
+        #     valid new_publics: public_read, private
+        #
+        # @param [String] new_public
+        # @return [String] new_public
+        #
         def public=(new_public)
+          if new_public
+            @acl = 'public-read'
+          else
+            @acl = 'private'
+          end
           new_public
         end
 
@@ -86,50 +132,32 @@ module Fog
         #     required attributes: directory, key
         #
         # @param expires [String] number of seconds (since 1970-01-01 00:00) before url expires
-        # @param options [Hash]
+        # @param options[Hash] No need to use
         # @return [String] url
         #
         def url(expires, options = {})
-
-          expires = expires.nil? ? 0 : expires.to_i
-
-          requires :directory, :key
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
-        end
-
-        def public_url
           requires :key
-          collection.get_url(key)
+          service.get_object_http_url_public(directory.key, key, expires)
         end
 
         def save(options = {})
           requires :body, :directory, :key
-          options['Content-Type'] = content_type if content_type
+          options['x-oss-object-acl'] ||= @acl if @acl
+          options['Cache-Control'] = cache_control if cache_control
           options['Content-Disposition'] = content_disposition if content_disposition
-          options.merge!(metadata_to_headers)
-          bucket_name, directory_key = collection.check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          if body.is_a?(::File)
-            service.put_object(object, body, options.merge(bucket: bucket_name))
-            data = service.head_object(object, bucket: bucket_name)
-          elsif body.is_a?(String)
-            data = service.put_object_with_body(object, body, options.merge(bucket: bucket_name)).data
+          options['Content-Encoding'] = content_encoding if content_encoding
+          options['Content-MD5'] = content_md5 if content_md5
+          options['Content-Type'] = content_type if content_type
+          options['Expires'] = expires if expires
+          options.merge!(metadata)
+
+          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+            multipart_save(options)
           else
-            raise Fog::Aliyun::Storage::Error, " Forbidden: Invalid body type: #{body.class}!"
+            service.put_object(directory.key, key, body, options)
           end
-          update_attributes_from(data)
-          refresh_metadata
-
+          self.etag = self.etag.gsub('"','') if self.etag
           self.content_length = Fog::Storage.get_body_size(body)
           self.content_type ||= Fog::Storage.get_content_type(body)
           true
@@ -137,70 +165,43 @@ module Fog
 
         private
 
-        attr_writer :directory
-
-        def refresh_metadata
-          metadata.reject! { |_k, v| v.nil? }
-        end
-
-        def headers_to_metadata
-          key_map = key_mapping
-          Hash[metadata_attributes.map { |k, v| [key_map[k], v] }]
-        end
-
-        def key_mapping
-          key_map = metadata_attributes
-          key_map.each_pair { |k, _v| key_map[k] = header_to_key(k) }
+        def directory=(new_directory)
+          @directory = new_directory
         end
 
-        def header_to_key(opt)
-          opt.gsub(metadata_prefix, '').split('-').map { |k| k[0, 1].downcase + k[1..-1] }.join('_').to_sym
-        end
-
-        def metadata_to_headers
-          header_map = header_mapping
-          Hash[metadata.map { |k, v| [header_map[k], v] }]
-        end
-
-        def header_mapping
-          header_map = metadata.dup
-          header_map.each_pair { |k, _v| header_map[k] = key_to_header(k) }
-        end
-
-        def key_to_header(key)
-          metadata_prefix + key.to_s.split(/[-_]/).map(&:capitalize).join('-')
-        end
+        def multipart_save(options)
+          # Initiate the upload
+          upload_id = service.initiate_multipart_upload(directory.key, key, options)
 
-        def metadata_attributes
-          if last_modified
-            bucket_name, directory_key = collection.check_directory_key(directory.key)
-            object = if directory_key == ''
-                       key
-                     else
-                       directory_key + '/' + key
-                     end
+          # Store ETags of upload parts
+          part_tags = []
 
-            data = service.head_object(object, bucket: bucket_name).data
-            if data[:status] == 200
-              headers = data[:headers]
-              headers.select! { |k, _v| metadata_attribute?(k) }
-            end
-          else
-            {}
+          # Upload each part
+          # TODO: optionally upload chunks in parallel using threads
+          # (may cause network performance problems with many small chunks)
+          # TODO: Support large chunk sizes without reading the chunk into memory
+          if body.respond_to?(:rewind)
+            body.rewind  rescue nil
+          end
+          while (chunk = body.read(multipart_chunk_size)) do
+            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk)
+            part_tags << part_upload
           end
-        end
 
-        def metadata_attribute?(key)
-          key.to_s =~ /^#{metadata_prefix}/
-        end
+          if part_tags.empty? #it is an error to have a multipart upload with no parts
+            part_upload = service.upload_part(directory.key, key, upload_id, 1, '')
+            part_tags << part_upload
+          end
 
-        def metadata_prefix
-          'X-Object-Meta-'
+        rescue
+          # Abort the upload & reraise
+          service.abort_multipart_upload(directory.key, key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
-        def update_attributes_from(data)
-          merge_attributes(data[:headers].reject { |key, _value| ['Content-Length', 'Content-Type'].include?(key) })
-        end
       end
     end
   end

data/lib/fog/aliyun/models/storage/files.rb

@@ -20,39 +20,13 @@ module Fog
 
         model Fog::Aliyun::Storage::File
 
-        # check_directory_key have two functions:
-        # 1. trim the directory_key suffix '/'
-        # 2. checking whether the directory_key is a bucket.
-        #    If so, it will return directly to avoid to create a new redundant folder named with directory_key.
-        #    This point will be applied to multi-bucket and make bucket as a directory scenario.
-        def check_directory_key(directory_key)
-          bucket_name = nil
-          if directory_key.is_a? Array
-            directory_key = directory_key[0]
-          end
-          if directory_key != ''
-            # trim the suffix '/'
-            directory_key = directory_key.chomp('/')
-            # The bucket name can not contain '/', so if directory_key, return directory.
-            if directory_key.include? '/'
-              directory_key
-            elsif service.bucket_exists?(directory_key)
-              bucket_name = directory_key
-              directory_key = ''
-            else
-              directory_key
-            end
-          end
-          return bucket_name, directory_key
-        end
-
         def all(options = {})
           requires :directory
           options = {
-              'delimiter'   => delimiter,
-              'marker'      => marker,
-              'max-keys'    => max_keys.to_i,
-              'prefix'      => prefix
+              'delimiter': delimiter,
+              'marker': marker,
+              'max-keys': max_keys.to_i,
+              'prefix': prefix
           }.merge!(options)
           options = options.reject {|key,value| value.nil? || value.to_s.empty?}
           merge_attributes(options)
@@ -68,7 +42,8 @@ module Fog
           end
         end
 
-        alias each_file_this_page each
+        alias_method :each_file_this_page, :each
+
         def each
           if !block_given?
             self
@@ -76,7 +51,7 @@ module Fog
             subset = dup.all
 
             subset.each_file_this_page { |f| yield f }
-            while subset.length == (subset.limit || 10_000)
+            while subset.is_truncated
               subset = subset.all(marker: subset.last.key)
               subset.each_file_this_page { |f| yield f }
             end
@@ -87,38 +62,16 @@ module Fog
 
         def get(key, options = {}, &block)
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
           begin
-            data = service.get_object(object, options.merge({bucket: bucket_name}), &block)
-            headers = data.headers
-            lastModified = headers[:last_modified]
-            last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-
-            date = headers[:date]
-            date = (Time.parse(date).localtime if !date.nil? && date != '')
-            file_data = {
-                body: data.body,
-                content_length: headers[:content_length].to_i,
-                key: key,
-                last_modified: last_modified,
-                content_type: headers[:content_type],
-                etag: headers[:etag],
-                date: date,
-                connection: headers[:connection],
-                accept_ranges: headers[:accept_ranges],
-                server: headers[:server],
-                object_type: headers[:x_oss_object_type]
-            }
-
+            data = service.get_object(directory.key, key, options, &block)
+            normalize_headers(data)
+            file_data = data.headers.merge({
+                                               :body => data.body,
+                                               :key  => key
+                                           })
             new(file_data)
-          rescue AliyunOssSdk::ServerError => error
-            case error.error_code
-            when %r{NoSuchKey},%r{SymlinkTargetNotExist}
+          rescue Exception => error
+            if error.respond_to?(:http_code) && error.http_code.to_i == 404
               nil
             else
               raise(error)
@@ -126,84 +79,50 @@ module Fog
           end
         end
 
-        def get_url(key)
+        # @param options[Hash] No need to use
+        def get_url(key, options = {})
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          service.get_object_http_url_public(object, 3600, bucket: bucket_name)
+          service.get_object_http_url_public(directory.key, key, 3600)
         end
 
+        # @param options[Hash] No need to use
         def get_http_url(key, expires, options = {})
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_http_url_public(object, expires, options.merge(bucket: bucket_name))
+          service.get_object_http_url_public(directory.key, key, expires)
         end
 
+        # @param options[Hash] No need to use
         def get_https_url(key, expires, options = {})
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          expires = expires.nil? ? 0 : expires.to_i
-          service.get_object_https_url_public(object, expires, options.merge(bucket: bucket_name))
+          service.get_object_https_url_public(directory.key, key, expires)
         end
 
-        def head(key, _options = {})
+        def head(key, options = {})
           requires :directory
-          bucket_name, directory_key = check_directory_key(directory.key)
-          object = if directory_key == ''
-                     key
-                   else
-                     directory_key + '/' + key
-                   end
-          data = service.head_object(object, bucket: bucket_name).data
-          return nil if data[:status] == 404
-          lastModified = data[:headers]['Last-Modified']
-          last_modified = (Time.parse(lastModified).localtime if !lastModified.nil? && lastModified != '')
-
-          date = data[:headers]['Date']
-          date = (Time.parse(date).localtime if !date.nil? && date != '')
-
-          file_data = {
-            content_length: data[:headers]['Content-Length'].to_i,
-            key: key,
-            last_modified: last_modified,
-            content_type: data[:headers]['Content-Type'],
-            etag: data[:headers]['ETag'],
-            date: date,
-            connection: data[:headers]['Connection'],
-            accept_ranges: data[:headers]['Accept-Ranges'],
-            server: data[:headers]['Server'],
-            object_type: data[:headers]['x-oss-object-type']
-          }
-          new(file_data)
-        rescue Fog::Aliyun::Storage::NotFound
-          nil
+          begin
+            data = service.head_object(directory.key, key, options)
+            normalize_headers(data)
+            file_data = data.headers.merge({
+                                               :key => key
+                                           })
+            new(file_data)
+          rescue Exception => error
+            if error.respond_to?(:http_code) && error.http_code.to_i == 404
+              nil
+            else
+              raise(error)
+            end
+          end
         end
 
         def new(attributes = {})
           requires :directory
-          # Sometimes, the v will be a Array, like "Prefix"=>[{}], "Marker"=>[xxxx], "MaxKeys"=>["100"], "IsTruncated"=>["false"]
-          # and there needs to parse them
-          for k, v in attributes
-            if !v.nil? && (v.is_a? Array) && (v.size > 0)
-              attributes[k] = v[0]
-            end
-          end
-          super({ directory: directory }.merge!(attributes))
+          super({ :directory => directory }.merge!(attributes))
+        end
+
+        def normalize_headers(data)
+          data.headers[:last_modified] = Time.parse(data.headers[:last_modified])
+          data.headers[:etag] = data.headers[:etag].gsub('"','')
         end
       end
     end

data/lib/fog/aliyun/requests/compute/attach_disk.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -46,7 +48,7 @@ module Fog
           if device
             parameters['Device'] = device
             pathUrl += '&Device='
-            pathUrl += URI.encode(device, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+            pathUrl += Addressable::URI.encode_component(device, Addressable::URI::CharacterClasses::UNRESERVED + '|')
           end
 
           signature = sign(@aliyun_accesskey_secret, parameters)

data/lib/fog/aliyun/requests/compute/create_security_group_egress_ip_rule.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -19,7 +21,7 @@ module Fog
 
           parameters['DestCidrIp'] = destCidrIp
           pathUrl += '&DestCidrIp='
-          pathUrl += URI.encode(destCidrIp, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(destCidrIp, Addressable::URI::CharacterClasses::UNRESERVED + '|')
           nicType ||= 'intranet'
           parameters['NicType'] = nicType
           pathUrl += '&NicType='
@@ -29,7 +31,7 @@ module Fog
           portRange ||= '-1/-1'
           parameters['PortRange'] = portRange
           pathUrl += '&PortRange='
-          pathUrl += URI.encode(portRange, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(portRange, Addressable::URI::CharacterClasses::UNRESERVED + '|')
 
           protocol = option[:protocol]
           protocol ||= 'all'

data/lib/fog/aliyun/requests/compute/create_security_group_egress_sg_rule.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -30,7 +32,7 @@ module Fog
           portRange ||= '-1/-1'
           parameters['PortRange'] = portRange
           pathUrl += '&PortRange='
-          pathUrl += URI.encode(portRange, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(portRange, Addressable::URI::CharacterClasses::UNRESERVED + '|')
 
           protocol = option[:protocol]
           protocol ||= 'all'

data/lib/fog/aliyun/requests/compute/create_security_group_ip_rule.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -19,7 +21,7 @@ module Fog
 
           parameters['SourceCidrIp'] = sourceCidrIp
           pathUrl += '&SourceCidrIp='
-          pathUrl += URI.encode(sourceCidrIp, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(sourceCidrIp, Addressable::URI::CharacterClasses::UNRESERVED + '|')
           nicType ||= 'intranet'
           parameters['NicType'] = nicType
           pathUrl += '&NicType='
@@ -29,7 +31,7 @@ module Fog
           portRange ||= '-1/-1'
           parameters['PortRange'] = portRange
           pathUrl += '&PortRange='
-          pathUrl += URI.encode(portRange, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(portRange, Addressable::URI::CharacterClasses::UNRESERVED + '|')
 
           protocol = option[:protocol]
           protocol ||= 'all'

data/lib/fog/aliyun/requests/compute/create_security_group_sg_rule.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -30,7 +32,7 @@ module Fog
           portRange ||= '-1/-1'
           parameters['PortRange'] = portRange
           pathUrl += '&PortRange='
-          pathUrl += URI.encode(portRange, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(portRange, Addressable::URI::CharacterClasses::UNRESERVED + '|')
 
           protocol = option[:protocol]
           protocol ||= 'all'

data/lib/fog/aliyun/requests/compute/create_vpc.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'addressable'
+
 module Fog
   module Compute
     class Aliyun
@@ -15,7 +17,7 @@ module Fog
 
           parameters['CidrBlock'] = cidrBlock
           pathUrl += '&CidrBlock='
-          pathUrl += URI.encode(cidrBlock, '/[^!*\'()\;?:@#&%=+$,{}[]<>`" ')
+          pathUrl += Addressable::URI.encode_component(cidrBlock, Addressable::URI::CharacterClasses::UNRESERVED + '|')
 
           name = options[:name]
           desc = options[:description]